/**
 * @module brainy
 * @description The `Brainy` class — the Triple-Intelligence database that unifies
 * vector similarity, graph traversal, and metadata filtering behind a single API
 * (`add`/`find`/`relate`/`related`/`get`/`similar`/`graph.*`/`now`/`asOf`/`transact`/
 * `export`/`import`). Native acceleration is feature-detected through the provider
 * boundary; when no provider is registered, the built-in JS engines serve everything.
 */

import { v4 as uuidv4, v7 as uuidv7 } from './universal/uuid.js'
import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from './utils/idNormalization.js'
import { JsHnswVectorIndex } from './hnsw/hnswIndex.js'
// TypeAwareHNSWIndex removed from default path — single unified JsHnswVectorIndex is faster
// for the 99% of queries that don't filter by type (avoids searching 42 separate graphs)
import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js'
import type { StorageOptions } from './storage/storageFactory.js'
import { rebuildCounts } from './utils/rebuildCounts.js'
import { jsonSafeIndexMetadata } from './utils/jsonSafeIndexMetadata.js'
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
import { BaseStorage } from './storage/baseStorage.js'
import {
  CURRENT_DATA_FORMAT,
  EXPECTED_INDEX_EPOCH,
  readBrainFormat,
  writeBrainFormat
} from './storage/brainFormat.js'
import type { BrainFormat } from './storage/brainFormat.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import { isZeroNormVector } from './utils/distance.js'
import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
import {
  defaultEmbeddingFunction,
  cosineDistance,
  getBrainyVersion
} from './utils/index.js'
import { embeddingManager } from './embeddings/EmbeddingManager.js'
import { matchesMetadataFilter, validateWhereFilter } from './utils/metadataFilter.js'
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js'
import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js'
import { VirtualFileSystem } from './vfs/VirtualFileSystem.js'
import { MetadataIndexManager } from './utils/metadataIndex.js'
import { detectContentType, extractForHighlighting } from './utils/contentExtractor.js'
import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js'
import {
  connectedComponents,
  stronglyConnectedComponents,
  pageRank,
  MinHeap
} from './graph/analyticsFallback.js'
import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js'
import { createPipeline } from './streaming/pipeline.js'
import { configureLogger, LogLevel, prodLog } from './utils/logger.js'
import { warnOnLowOsLimits } from './utils/osLimits.js'
import { setGlobalCache } from './utils/unifiedCache.js'
import type { UnifiedCache } from './utils/unifiedCache.js'
import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js'
import { PluginRegistry, isVersionedIndexProvider, isGraphAccelerationProvider } from './plugin.js'
import type {
  GraphAccelerationProvider,
  TraverseOptions,
  Subgraph,
  RankOptions,
  CommunitiesOptions,
  PathOptions,
  MetadataIndexProvider,
  OpaqueIdSet,
  AtGenerationVectors,
  VectorIndexProvider,
  GraphIndexProvider,
  ProviderMaintenanceDebt
} from './plugin.js'
import type {
  BrainyPlugin,
  BrainyPluginContext,
  GraphCompressionProvider
} from './plugin.js'
import { ConnectionsCodec } from './hnsw/connectionsCodec.js'
import { TransactionManager } from './transaction/TransactionManager.js'
import { transactTimeoutBudget } from './transaction/Transaction.js'
import { RevisionConflictError } from './transaction/RevisionConflictError.js'
import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js'
import {
  ValidationConfig,
  validateAddParams,
  validateUpdateParams,
  validateRelateParams,
  validateUpdateRelationParams,
  validateFindParams,
  recordQueryPerformance
} from './utils/paramValidation.js'
import { findCallerLocation } from './utils/callerLocation.js'
import {
  SaveNounMetadataOperation,
  SaveNounOperation,
  AddToVectorIndexOperation,
  AddToMetadataIndexOperation,
  SaveVerbMetadataOperation,
  SaveVerbOperation,
  AddToGraphIndexOperation,
  RemoveFromVectorIndexOperation,
  ReplaceInVectorIndexOperation,
  RemoveFromMetadataIndexOperation,
  RemoveFromGraphIndexOperation,
  UpdateNounMetadataOperation,
  UpdateVerbMetadataOperation,
  DeleteNounMetadataOperation,
  DeleteVerbMetadataOperation
} from './transaction/operations/index.js'
import {
  BaseOperationalMode,
  ReaderMode,
  HybridMode
} from './storage/operationalModes.js'
import {
  Entity,
  Relation,
  Result,
  AddParams,
  UpdateParams,
  RelateParams,
  UpdateRelationParams,
  FindParams,
  SimilarParams,
  RelatedParams,
  GraphApi,
  GraphView,
  GraphNode,
  SubgraphOptions,
  SubgraphSelector,
  GraphExportOptions,
  GraphRankOptions,
  GraphRankEntry,
  GraphCommunitiesOptions,
  GraphCommunitiesResult,
  GraphPathOptions,
  GraphPathResult,
  GetOptions,
  AddManyParams,
  RemoveManyParams,
  UpdateManyParams,
  RelateManyParams,
  BatchResult,
  BrainyConfig,
  BrainyStats,
  ScoreExplanation,
  FillSubtypeRule,
  FillSubtypeRules,
  FillSubtypesResult,
  RepairReport,
  RepairFamilyReport
} from './types/brainy.types.js'
import { NounType, VerbType, TypeUtils } from './types/graphTypes.js'
import {
  splitNounMetadataRecord,
  splitVerbMetadataRecord,
  buildNounMetadataRecord,
  buildVerbMetadataRecord
} from './types/reservedFields.js'
import { BrainyInterface } from './types/brainyInterface.js'
import type { IntegrationHub, IntegrationHubConfig } from './integrations/core/IntegrationHub.js'
import { MigrationRunner } from './migration/MigrationRunner.js'
import type { MigrationPreview, MigrationResult, MigrateOptions } from './migration/types.js'
import { AggregationIndex } from './aggregation/AggregationIndex.js'
import { AggregateMaterializer } from './aggregation/materializer.js'
import type { AggregateDefinition, AggregateQueryParams, AggregateResult } from './types/brainy.types.js'
import type { MigrationProgress } from './types/brainy.types.js'
import type { IndexedProjectionPath, WaitForIndexedOptions } from './types/brainy.types.js'
import { WaitForIndexedTimeoutError } from './types/brainy.types.js'
import { resolveJsHnswConfig, DEFAULT_RECALL } from './utils/recallPreset.js'
import * as fs from 'node:fs'
import * as os from 'node:os'
import { Db, type DbHost, type HistoricalQueryHandle } from './db/db.js'
import { entityMatchesFind } from './db/whereMatcher.js'
import {
  importGraph,
  isPortableGraph,
  type PortableGraph,
  type ExportSelector,
  type ExportOptions,
  type ImportOptions,
  type ImportResult
} from './db/portableGraph.js'
import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js'
import type { FactScanHandle, FactMarkerRecord } from './db/factLog.js'
import {
  ENTITY_TREE_STAMP_PATH,
  readFamilyStamp,
  verifyFamilyStamp,
  writeFamilyStamp,
  type FamilyStamp
} from './db/familyStamp.js'
import {
  ChangeFeed,
  type BrainyChangeEvent,
  type ChangeListener,
  type PendingChangeEvent
} from './events/changeFeed.js'
import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js'
import { GenerationConflictError, StoreInconsistentError } from './db/errors.js'
import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js'
import {
  assessIndexReadiness,
  assessProviderHealth,
  assessProviderRebuild,
  describeRebuildProgress
} from './utils/indexReadiness.js'
import { reconstructNounWrapper } from './db/factLog.js'
import { asBrainyFieldRefusal } from './db/fieldAddressing.js'
import {
  readLogAuthority,
  runLogCompletenessOracle,
  flipToLogAuthority,
  recordDigest,
  nounEntityTruth,
  LOG_AUTHORITY_PATH,
  type LogAuthorityRecord,
  type LogAuthorityStorage,
  type OracleReport
} from './db/logAuthority.js'
import { MemoryStorage } from './storage/adapters/memoryStorage.js'
import type {
  CompactHistoryOptions,
  CompactHistoryResult,
  HistoryStats,
  TransactOptions,
  TransactReceipt,
  TxLogEntry,
  TxOperation,
  DiffResult,
  EntityHistory,
  HistoryVersion
} from './db/types.js'
import { stableDeepEqual } from './db/stableEqual.js'
import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js'
import type { Operation, TransactionFunction } from './transaction/types.js'

/**
 * Stopwords for semantic highlighting
 * These common words are skipped when highlighting individual words
 * to focus on meaningful content words.
 */
const STOPWORDS = new Set([
  'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been',
  'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should',
  'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in', 'for', 'on', 'with', 'at',
  'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
  'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
  'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such',
  'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just',
  'and', 'but', 'or', 'if', 'because', 'until', 'while', 'although', 'though',
  'this', 'that', 'these', 'those', 'it', 'its', 'i', 'me', 'my', 'you', 'your',
  'he', 'him', 'his', 'she', 'her', 'we', 'us', 'our', 'they', 'them', 'their',
  'what', 'which', 'who', 'whom', 'whose', 'am'
])

/**
 * Optional capabilities a plugin-provided `'vector'` engine may expose beyond
 * the built-in {@link JsHnswVectorIndex} surface ({@link VectorIndexProvider}
 * declares neither). Every call site feature-detects with
 * `typeof x === 'function'` before invoking, so an engine without these hooks
 * is fully supported.
 */
interface VectorIndexOptionalHooks {
  /** Switch persistence mode at runtime (bulk-ingest optimization). */
  setPersistMode?: (mode: 'immediate' | 'deferred') => void
  /** Release engine resources (timers, file handles, native memory). */
  close?: () => Promise<void> | void
}

/**
 * Optional lifecycle hook a plugin-provided `'metadataIndex'` implementation
 * may expose beyond the built-in {@link MetadataIndexManager} surface.
 * Feature-detected with `typeof x === 'function'` before invoking.
 */
interface MetadataIndexOptionalHooks {
  /** Release index resources (timers, native memory). */
  close?: () => Promise<void> | void
}

/**
 * Brainy's internal resolved configuration. `normalizeConfig()` defaults every
 * field at construction except the genuinely-optional ones listed in the
 * `Pick`, which legitimately stay `undefined` when the consumer omitted them —
 * every read site handles that explicitly (`?.` chains and
 * `!== undefined` guards).
 */
type ResolvedBrainyConfig = Required<
  Omit<
    BrainyConfig,
    | 'maxQueryLimit'
    | 'reservedQueryMemory'
    | 'vector'
    | 'plugins'
    | 'integrations'
    | 'retention'
    | 'eagerEmbeddings'
    | 'migrationWaitTimeoutMs'
    | 'transactionBudgetFloorMs'
    | 'persistence'
  >
> &
  Pick<
    BrainyConfig,
    | 'maxQueryLimit'
    | 'reservedQueryMemory'
    | 'vector'
    | 'plugins'
    | 'integrations'
    | 'retention'
    | 'eagerEmbeddings'
    | 'migrationWaitTimeoutMs'
    | 'transactionBudgetFloorMs'
    | 'persistence'
  >

/**
 * Result type for brain.diagnostics()
 */
export interface DiagnosticsResult {
  version: string
  plugins: { active: string[], count: number }
  providers: Record<string, { source: 'plugin' | 'default' }>
  indexes: {
    hnsw: { size: number, type: string }
    metadata: { type: string, initialized: boolean }
    graph: { type: string, initialized: boolean, wiredToStorage: boolean }
  }
}

/**
 * In-flight planning state for one `brain.transact()` batch (8.0 MVCC).
 *
 * Operations inside a batch may reference ids written by EARLIER operations
 * of the same batch (`add` an entity, then `relate` to it) — but nothing has
 * touched storage yet during planning. This state carries the
 * would-be-written stored objects so later planners read
 * "current state + batch so far", exactly what the executed batch produces.
 */
interface TxPlanState {
  /** Stored metadata + vector of entities written earlier in this batch. */
  nouns: Map<string, { metadata: Record<string, any>; vector: Vector }>
  /** Entity ids removed earlier in this batch. */
  removedNouns: Set<string>
  /** Verbs created earlier in this batch (keyed by relationship id). */
  verbs: Map<string, GraphVerb>
  /** Relationship ids removed earlier in this batch. */
  removedVerbs: Set<string>
}

/**
 * The fully planned form of one `brain.transact()` batch: the ordered
 * `TransactionManager` operations, the resolved id per input operation (the
 * receipt), the touched-id sets the generation store stages before-images
 * for, and the post-commit hooks (aggregation-index maintenance — derived
 * data, applied outside the atomic batch exactly as the single-op methods
 * do).
 */
interface PlannedTransact {
  /** Ordered operations for one atomic TransactionManager execution. */
  operations: Operation[]
  /** Resolved id per input operation, in input order. */
  ids: string[]
  /** Entity ids the batch writes or deletes. */
  touchedNouns: string[]
  /** Relationship ids the batch writes or deletes. */
  touchedVerbs: string[]
  /** Aggregation-index hooks to run after the commit point. */
  postCommit: Array<() => void>
  /**
   * One entry per staged `{ op: 'update' }`, re-verified UNDER the commit
   * mutex (the generation store's `precommit`) so per-op `ifRev` CAS is
   * atomic with the apply — planning-time checks can interleave with
   * concurrent writers. `updatedMetadata` is the staged metadata object (held
   * by reference by the staged operation), re-stamped there with the
   * authoritative `_rev`. A conflict rejects the WHOLE batch.
   */
  casUpdates: Array<{
    id: string
    ifRev?: number
    updatedMetadata: { _rev?: number }
  }>
  /**
   * Ids whose revision baseline THIS batch resets via an `add` op (a fresh
   * create, or add's overwrite semantics which restamp `_rev: 1`). Updates to
   * these ids sequence against in-batch state no concurrent writer can touch,
   * so their plan-time CAS checks and rev stamps are already exact — the
   * commit precondition leaves them untouched.
   */
  createdNouns: Set<string>
  /**
   * Change-feed events, one per affected record in op order, populated by the
   * planners ONLY when a listener is subscribed. Stamped with the batch's
   * committed generation and emitted after `commitTransaction` returns — a
   * rejected batch (CAS conflict, failed apply) emits nothing.
   */
  changeEvents: PendingChangeEvent[]
  /**
   * V2 marker records riding the batch's ONE commit fact (e.g. the
   * deferred-embedding pending markers) — same generation, same atomic
   * append as the batch itself. A rejected batch appends no fact, so no
   * marker outlives its write.
   */
  markerRecords: FactMarkerRecord[]
  /**
   * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to
   * decrement on the vectored-noun ledger — consumed by `transact()` with a
   * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER
   * `commitTransaction` resolves (never for a rejected batch). Kept separate
   * from `postCommit` (`Array<() => void>`, called synchronously, fire-and-
   * forget) because the ledger hook is async and must be awaited.
   */
  vectorUnlands: string[]
}

/**
 * Internal control-flow signal: an insert guarded by a must-be-absent
 * precondition (`add({ ifAbsent })` / `add({ upsert })`) found the entity
 * already present in the authoritative before-image under the commit mutex —
 * a concurrent writer created it after the planning-time absence check.
 * Never escapes `add()`: the caller converts it into the documented
 * resolution (ifAbsent → return the existing id without writing; upsert →
 * merge into the now-existing entity via `update()`).
 */
class InsertPreconditionExistsSignal extends Error {
  constructor(readonly id: string) {
    super(`insert precondition: entity ${id} already exists`)
    this.name = 'InsertPreconditionExistsSignal'
  }
}

/**
 * @description The derived-index families a read may depend on. A read that
 * consults none of them (a canonical-storage read: `get`, an entity
 * enumeration, a VFS content/dir read) is index-independent and must never
 * block on another family's one-time migration. Used by the family-scoped
 * migration gate ({@link Brainy.awaitMigrationLock}).
 */
export type IndexFamily = 'vector' | 'metadata' | 'graph'

/**
 * @description Honest per-surface outcome for {@link Brainy.warm}. Literal
 * meanings — never conflate the first two:
 * - `'warmed'` — the provider's own `warm?()` hook ran (vector/graph), or the
 *   surface's full-hydration seam loaded EVERY shard/field/segment from
 *   storage (metadata; graph's fallback path). The surface is genuinely at
 *   steady-state cost for the next operation.
 * - `'probed'` — no `warm?()` hook was available, so a best-effort read
 *   (e.g. one `search()` call) faulted in *some* backing storage as a side
 *   effect. Real work happened, but it is NOT the same guarantee as
 *   `'warmed'` — never reported as `'warmed'`.
 * - `'unavailable'` — nothing ran: no hook, no hydration seam, and (for the
 *   vector probe fallback) nothing to probe (an empty index or unknown
 *   vector dimension). The surface is unchanged by this `warm()` call.
 */
export type WarmOutcome = 'warmed' | 'probed' | 'unavailable'

/**
 * @description Result of {@link Brainy.warm}: one {@link WarmOutcome} +
 * elapsed time per index surface, plus the total wall-clock time for the
 * whole call. `durationMs` is measured around exactly the work described by
 * that surface's `outcome` (e.g. the vector entry's `durationMs` times the
 * provider `warm()` call OR the probe `search()` call — whichever ran).
 */
export interface WarmReport {
  vector: { outcome: WarmOutcome; durationMs: number }
  metadata: { outcome: WarmOutcome; durationMs: number }
  graph: { outcome: WarmOutcome; durationMs: number }
  /** Total wall-clock time for the whole `warm()` call (all three surfaces). */
  totalDurationMs: number
}

/**
 * @description Result of {@link Brainy.maintenanceDebt}: one outcome per
 * index surface, mirroring {@link WarmReport}'s shape.
 * - `'reported'` — the active provider for this surface implements
 *   `maintenanceDebt?()` and its {@link ProviderMaintenanceDebt} payload is
 *   attached verbatim under `debt`.
 * - `'unavailable'` — the active provider does not implement the hook, so
 *   nothing is known; brainy never estimates or infers a payload on its
 *   behalf.
 */
export type MaintenanceDebtOutcome = 'reported' | 'unavailable'

/**
 * @description Per-surface result of {@link Brainy.maintenanceDebt}. Brainy
 * performs no thresholding, polling, or estimation over this data — it is a
 * pure passthrough of each active provider's own self-report (the provider
 * owns the numbers; the operator owns the policy).
 */
export interface MaintenanceDebtReport {
  vector: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt }
  metadata: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt }
  graph: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt }
}

/**
 * How long a failed aggregation-backfill walk suppresses fresh walk attempts.
 * Within the window, queries rethrow the recorded failure instantly (loud,
 * cheap); after it, one new attempt is allowed. Bounds the damage of a
 * caller-side tight retry loop against a deterministically-failing store.
 */
const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000

/**
 * Time budget for the auto-compaction pass at close() (8.9.0). Bounds how long
 * a clean shutdown spends reclaiming history backlog — an early stop is a
 * consistent prefix and the next close/explicit pass resumes. Explicit
 * `compactHistory()` calls are unbounded unless the caller passes their own
 * `timeBudgetMs` (maintenance windows choose their own budgets).
 */
const CLOSE_COMPACTION_BUDGET_MS = 5_000

/**
 * The main Brainy class - Clean, Beautiful, Powerful
 * REAL IMPLEMENTATION - No stubs, no mocks
 *
 * Implements BrainyInterface to ensure consistency across integrations
 */
export class Brainy<T = any> implements BrainyInterface<T> {
  // Static shutdown hook tracking (global, not per-instance)
  private static shutdownHooksRegisteredGlobally = false
  private static instances: Brainy[] = []

  /** The globally-registered shutdown listeners, kept so the LAST close() can
   *  deregister them. A process.on('SIGINT'/'SIGTERM') listener holds a ref'd
   *  signal handle that keeps the Node event loop alive — a library that never
   *  removes its listeners makes every bare script hang after close().
   *  See {@link registerShutdownHooks} / {@link deregisterShutdownHooksIfIdle}. */
  private static sigtermListener?: () => void
  private static sigintListener?: () => void
  private static beforeExitListener?: () => void

  /** Poll cadence (ms) for the migration LOCK when a provider exposes no
   *  event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */
  private static readonly MIGRATION_POLL_INTERVAL_MS = 250

  /** First-party accelerator packages probed by guarded auto-detection when
   *  `plugins` is undefined (installing one IS the opt-in). See {@link loadPlugins}. */
  private static readonly AUTO_DETECT_PLUGIN_PACKAGES = ['@soulcraft/cor']

  // Core components
  private index!: JsHnswVectorIndex
  private storage!: BaseStorage
  private metadataIndex!: MetadataIndexManager
  private graphIndex!: GraphAdjacencyIndex
  private transactionManager: TransactionManager

  /**
   * 8.0 generational MVCC record layer (created in `init()`, before any index
   * is built — crash recovery may rewrite canonical entity files). Owns the
   * generation counter, the transact commit protocol, pin refcounts, and
   * point-in-time record resolution. Backs `now()`/`transact()`/`asOf()`/
   * `compactHistory()` and the `Db` value type.
   */
  private generationStore!: GenerationStore
  /**
   * Resolves the in-flight write generation as a BigInt, for stamping graph
   * edges via the {@link GraphIndexProvider} contract. Passed to every
   * `AddToGraphIndexOperation`/`RemoveFromGraphIndexOperation` and evaluated
   * when the operation executes — inside a `transact()` batch the generation
   * store has assigned the batch generation by then; for single-op writes it
   * reads the post-write watermark. The arrow body reads `generationStore`
   * lazily, so it is safe to define before `init()` assigns the store.
   * Metadata/vector index writes use the bootstrap-honest twin
   * {@link indexWriteGeneration} below.
   */
  private readonly graphWriteGeneration = (): bigint =>
    BigInt(this.generationStore.generation())
  /**
   * The metadata/vector twin of {@link graphWriteGeneration}, honest about
   * bootstrap: while generation stamping is inactive (init-time
   * infrastructure writes, e.g. the VFS root, applied via
   * `runWithoutGeneration`) there IS no commit generation — this resolves to
   * `undefined` so a provider records "unstamped", never a fabricated 0.
   * The graph thunk keeps its non-optional `bigint` contract (no graph
   * writes occur during bootstrap).
   */
  private readonly indexWriteGeneration = (): bigint | undefined =>
    this._generationStampingActive
      ? BigInt(this.generationStore.generation())
      : undefined
  /** Lazily built host surface shared by every `Db` value of this brain. */
  private _dbHost?: DbHost<T>
  /**
   * GC backstop for leaked `Db` pins: when an unreleased `Db` is collected,
   * its generation pin (store + versioned providers) is released and an
   * owned snapshot brain (from `Brainy.load()`) is closed. Explicit
   * `db.release()` unregisters first, so pins are never double-released.
   */
  private _dbFinalizationRegistry?: FinalizationRegistry<{
    generation: number
    closeOnRelease?: () => Promise<void>
  }>
  private embedder: EmbeddingFunction
  private distance: DistanceFunction
  private config: ResolvedBrainyConfig

  /**
   * Bounded warm cache for verb-int → verb-id resolution (8.0 u64 contract).
   * Fed by `addVerb` returns and `verbIntsToIds` results; evicted in insertion
   * order once {@link Brainy.VERB_INT_WARM_CACHE_MAX} is exceeded. Pure
   * optimization — the graph-index provider owns the durable interning, so a
   * miss just means one extra `verbIntsToIds` round trip.
   */
  private readonly verbIntWarmCache = new Map<bigint, string>()
  /** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */
  private static readonly VERB_INT_WARM_CACHE_MAX = 100_000

  /**
   * Default cap on how many `find()` matches seed a `graph.subgraph(query)` when the
   * caller pins no explicit `limit` — bounds the materialized seed set on the JS /
   * non-opaque path (the native opaque path forwards the whole universe, uncapped).
   */
  private static readonly QUERY_SEED_CAP = 10_000

  // Silent mode state
  private originalConsole?: {
    log: typeof console.log
    info: typeof console.info
    warn: typeof console.warn
    error: typeof console.error
  }

  // Plugin system
  private pluginRegistry = new PluginRegistry()
  /**
   * Resolved native graph-acceleration provider, cached on first `brain.graph.*`
   * access. `undefined` = not yet resolved; `null` = resolved, none registered
   * (use the TS fallback); otherwise the provider instance.
   */
  private _graphAccelProvider: GraphAccelerationProvider | null | undefined = undefined
  /**
   * Set when a non-fatal index rebuild fails at init() (the brain is allowed to
   * start, but in a degraded state where queries may be incomplete). Surfaced
   * through {@link checkHealth} so consumers can observe the failure
   * programmatically rather than only via the console. A storage *read* failure
   * during rebuild is NOT recorded here — it re-throws and aborts init() loudly.
   */
  private _indexRebuildFailed: Error | null = null
  /**
   * Ids of records that committed via an adopt-forward failed-rollback recovery
   * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is
   * durable but its derived index entry may be incomplete until
   * {@link repairIndex}. Non-empty = a queryable degraded state, folded into
   * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read
   * paths. Cleared by {@link repairIndex}.
   */
  private _indexDegradedIds: Set<string> = new Set()
  /** One-shot guard so the degraded-reads warning fires once per degraded window
   *  (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */
  private _degradedReadWarned = false
  /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */
  private _graphAdjacencyVerified = false
  /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */
  private _graphAdjacencyVerifying = false
  /** Metadata field-index cold-read guard: verified-serving this session (one-shot). */
  private _metadataVerified = false
  /** Re-entrancy guard for {@link verifyMetadataLive}. */
  private _metadataVerifying = false
  /** Vector-index cold-read guard: verified-serving this session (one-shot). */
  private _vectorVerified = false
  /** Re-entrancy guard for {@link verifyVectorLive}. */
  private _vectorVerifying = false
  /**
   * Coordinated migration LOCK (#18): dedup guards so the "upgrading, blocking"
   * and "upgrade complete, resumed" lines each log once per migration window,
   * not once per blocked operation. See {@link awaitMigrationLock}.
   */
  private _migrationBlockLogged = false
  private _migrationReleaseLogged = false
  /** Epoch ms when brainy first observed the migration LOCK held, or `null` when
   *  not migrating — the fallback `elapsedMs` a provider that omits `migrationStatus()`
   *  timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */
  private _migrationObservedAt: number | null = null
  /** Path of the pre-upgrade backup taken this open (default-on, opt-out via
   *  `migrationBackup: false`), or `null` if none. Removed once the 7.x→8.0
   *  upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */
  private _migrationBackupPath: string | null = null

  /**
   * The in-process change feed behind {@link onChange}. Emitted from the
   * commit seam ({@link persistSingleOp} / {@link transact}), so every
   * canonical mutation — any origin — produces exactly one post-commit event
   * per affected record. See src/events/changeFeed.ts for the delivery
   * contract.
   */
  private readonly _changeFeed = new ChangeFeed()

  /** Set when the on-open VFS-blob adoption left one or more `_cow/` blobs it
   *  could not fully adopt (bytes or metadata missing). While true, the
   *  pre-upgrade backup is NOT auto-removed — the upgrade is not verifiably
   *  complete for the VFS. See {@link autoAdoptLegacyVfsBlobsIfNeeded}. */
  private _vfsBlobAdoptionIncomplete = false

  /**
   * 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat}
   * marker (`_system/brain-format.json`) as read during the store-open phase,
   * or `null` for a pre-handshake / brand-new brain. Populated BEFORE any
   * derived index or native provider is constructed, so {@link formatInfo}
   * answers synchronously when the provider reads it at its own init().
   * Refreshed to the current marker once it is (re)stamped.
   */
  private _brainFormat: BrainFormat | null = null
  /**
   * True when the on-disk {@link _brainFormat} is absent OR carries an
   * `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived
   * JS indexes on disk predate this build and must be rebuilt from the
   * canonical records (the epoch-drift rebuild trigger that closes the gap
   * where JS indexes rebuilt only on `size()===0`, never on a format change).
   * Cleared once the rebuild verifies and the marker is re-stamped.
   */
  private _indexEpochStale = false

  // Sub-APIs (lazy-loaded)
  private _nlp?: NaturalLanguageProcessor
  private _extractor?: NeuralEntityExtractor
  private _tripleIntelligence?: TripleIntelligenceSystem
  private _vfs?: VirtualFileSystem
  private _vfsInitialized = false  // Track VFS init completion separately
  /**
   * 8.0 MVCC: single-op writes create generations (Model-B) only AFTER init
   * completes. Init-time infrastructure writes (the VFS root directory, etc.)
   * form the generation-0 baseline of a freshly-materialized brain and are NOT
   * versioned — so a brand-new brain reports `generation() === 0` / empty
   * `transactionLog()`, and the first USER write is generation 1. Enabled at
   * the tail of `init()`; see {@link persistSingleOp}.
   */
  private _generationStampingActive = false
  /**
   * 8.0 MVCC: the coordinator-driven adaptive history byte budget for this
   * brain (set via {@link setRetentionBudget}; e.g. cor's `ResourceManager`
   * fair-shares one box-wide budget across co-located instances). Overrides the
   * local `os.freemem` probe while `retention` is adaptive. `undefined` = use
   * the probe. See {@link adaptiveHistoryBudgetBytes}.
   */
  private _retentionBudgetBytes?: number
  private _hub?: IntegrationHub     // Integration Hub for external tools
  private _pendingMigrationRunner?: MigrationRunner  // Deferred migration runner for large datasets
  private _aggregationIndex?: AggregationIndex       // Incremental aggregation engine
  private _aggregationBackfillFlight: Promise<void> | null = null  // Single-flight backfill walk
  private _aggregationCatchUpFlight: Promise<void> | null = null   // Single-flight behind-stamp catch-up

  // ENGINE-OWNED PERSISTENCE CADENCE (SELF-ENGINE-LIFECYCLE-SPRINT):
  // write-count / interval / idle triggers → ONE background flush at a time.
  // Write acks NEVER await it; a failed background flush is LOUD and re-armed.
  private _persistDirtyWrites = 0
  private _persistLastFlushAt = Date.now()
  /**
   * Whether a write has been committed since the last flush that ran. THE
   * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written
   * to has nothing to make durable, and a flush over it must cost nothing and
   * say nothing. Before this, a flush called every provider, stamped the
   * watermarks, persisted the generation counter and re-stamped the entity
   * tree whether or not anything had changed — roughly 28 writes for a store
   * that had not moved.
   *
   * WHAT THIS DOES NOT EXPLAIN, stated so nobody reads it as solved: a
   * production process holding 21 brains printed "All indexes flushed to disk
   * in 216-601ms" per brain every ~35s and idled at 1.26 cores with no writes
   * for ten minutes. This engine's cadence is WRITE-DRIVEN — every trigger
   * runs through noteWriteForPersistence, which only a committed write calls —
   * so something was calling flush() on those brains, and this gate makes such
   * a call free rather than accounting for it. The caller is still unidentified.
   */
  private _dirtySinceLastFlush = false
  private _persistIdleTimer: ReturnType<typeof setTimeout> | null = null
  private _persistBackgroundFlight: Promise<void> | null = null

  // DEFERRED EMBEDDING (MT5): pending markers are LOG RECORDS — an
  // embed.pending record rides the deferred write's own commit fact and
  // embed.landed rides the landing commit; this set is the in-memory
  // fast-path index, rebuilt at open by folding the log's marker records.
  // ONE background worker drains it. A crash can delay a vector, never
  // lose one.
  private _pendingEmbedIds = new Set<string>()
  private _embedWorkerFlight: Promise<void> | null = null

  // OPEN-PATH FIX: the background embedding-engine warm kicked off (never
  // awaited) by `performInit()` when `eagerEmbeddings` resolves true. Stored
  // for observability only — `embed()`/`embeddingManager.embed()` already
  // await the engine's OWN singleton init promise internally, so nothing
  // needs to explicitly await this field for correctness. Never rejects on
  // its own: a `.catch` narrates the failure and swallows it so a failed
  // warm never surfaces as an unhandled rejection.
  private _embeddingWarmPromise: Promise<void> | null = null

  /** The stored log-authority switch, read once at open (default: tree). */
  private _logAuthority: LogAuthorityRecord = { authority: 'tree' }
  // A failed walk latches its error: retries within the cooldown rethrow it
  // instantly instead of re-walking, so a tight caller-side retry loop costs
  // one loud error per query, never a full store walk per query.
  private _aggregationBackfillFailure: { at: number; error: Error } | null = null
  private _materializer?: AggregateMaterializer      // Debounced materialization of aggregate results
  /**
   * Fields registered via `brain.trackField()` — drives optional value validation on
   * `add()`/`update()` and powers `brain.counts.byField()`. Outer key is the field
   * name (`'status'`, `'paradigm'`, `'role'`, ...); inner record carries the per-NounType
   * flag and an optional value whitelist (when provided, writes with off-vocabulary
   * values are rejected).
   */
  private _trackedFields: Map<string, { perType: boolean; values?: Set<string> }> = new Map()

  /**
   * Per-NounType / per-VerbType subtype enforcement rules registered via
   * `brain.requireSubtype(type, options)`. When `required` is true the matching
   * write path throws if subtype is missing; when `values` is set the matching
   * write path throws if the value is off-vocabulary. Composes with the
   * brain-wide `requireSubtype` constructor flag.
   */
  private _requiredSubtypes: Map<string, { required: boolean; values?: Set<string> }> = new Map()

  // State
  private initialized = false
  private dimensions?: number

  // Multi-process mode (writer is default; reader is set via mode: 'reader' or
  // Brainy.openReadOnly()). `operationalMode.validateOperation('write')` is
  // called at the top of every mutation method. Snapshots from `asOf()` also
  // set this to ReaderMode so historical instances are protected the same way.
  private operationalMode: BaseOperationalMode

  // Write-quarantine after a failed transaction rollback left the store
  // inconsistent (see StoreInconsistentError). Set at the moment the failure is
  // surfaced; every subsequent mutation is refused via assertWritable until
  // repairIndex() reconciles canonical vs derived state and clears it. Reads are
  // never affected. null = healthy.
  private storeInconsistency: StoreInconsistentError | null = null

  // Ready Promise state (Unified readiness API)
  // Allows consumers to await brain.ready for initialization completion
  private _readyPromise: Promise<void> | null = null
  private _readyResolve: (() => void) | null = null
  private _readyReject: ((error: Error) => void) | null = null

  // In-flight init() promise. Guards against concurrent initialization when
  // multiple operations race to lazily initialize the same instance — all
  // racers await the single in-flight run. Reset on failure so init() can
  // be retried after the underlying cause (e.g. a held writer lock) clears.
  private initPromise: Promise<void> | null = null

  // Set once close() has torn the instance down. close() is terminal: a closed
  // instance must NOT silently lazy-re-initialize on the next operation (that
  // would resurrect released indexes and timers). The lazy-init convenience
  // applies only to instances that were never closed.
  private closed = false

  // Index-build-at-open state. `lazyRebuildCompleted` predates the health-gate
  // law (it named a first-QUERY lazy rebuild) and stays for `getIndexStatus()`
  // API compatibility, but its truth changed: a needed rebuild now runs
  // unconditionally at open() (see `rebuildIndexesIfNeeded`), never deferred to
  // a read, so this simply flips true once that open-time step has run.
  // `lazyRebuildInProgress` / `lazyRebuildPromise` (the first-query rebuild's
  // concurrency guard) are retired with the lazy-build path they served —
  // `ensureIndexesLoaded()` is a read-time CHECK now, never a build.
  private lazyRebuildCompleted = false

  // Read-gate narration dedup: a degraded-but-serving or not-ready health
  // report narrates via prodLog.warn ONCE per (provider, report.generation) —
  // never once per read. Keyed on the provider instance itself.
  /**
   * The last health narration emitted per provider, keyed by its CONTENT.
   *
   * This used to dedupe on the provider's `generation` counter, which bumps on
   * every ledger mutation and every rebuild boundary — so a provider that
   * bumps its generation on routine work re-emitted the same unchanged health
   * line on every read that consulted it, and a provider that never bumped
   * could suppress a line whose reasons had genuinely changed. The dedupe key
   * is now what the line SAYS: an unchanged verdict is silent however the
   * generation moves, and a changed verdict is always heard.
   */
  private _lastNarratedHealth = new Map<unknown, string>()

  constructor(config?: BrainyConfig) {
    // The reserved-field write policy died with the field-addressing law:
    // every metadata name is the user's now (engine scalars write via their
    // dedicated params and read at `system.*`), so there is nothing left for
    // the policy to govern. A config still passing it refuses loudly rather
    // than being silently ignored.
    if (config && 'reservedFieldPolicy' in (config as Record<string, unknown>)) {
      throw new Error(
        `reservedFieldPolicy was removed by the field-addressing law: metadata field ` +
          `names are never reserved anymore — every name in the metadata bag is the ` +
          `user's and works like any other field. Set engine scalars via their ` +
          `dedicated params (confidence, weight, subtype, …) and query them as ` +
          `system.<field>. Remove the reservedFieldPolicy option.`
      )
    }

    // Normalize configuration with defaults
    this.config = this.normalizeConfig(config)

    // Multi-process mode — default 'writer' uses HybridMode (read + write),
    // 'reader' uses ReaderMode (read-only, all mutations throw). A writer needs
    // to read its own data, so the writer role is read-write rather than
    // write-only.
    this.operationalMode = this.config.mode === 'reader'
      ? new ReaderMode()
      : new HybridMode()

    // Configure memory limits
    // This must happen early, before any validation occurs
    if (this.config.maxQueryLimit !== undefined || this.config.reservedQueryMemory !== undefined) {
      ValidationConfig.reconfigure({
        maxQueryLimit: this.config.maxQueryLimit,
        reservedQueryMemory: this.config.reservedQueryMemory
      })
    }

    // Setup core components
    this.distance = cosineDistance
    this.embedder = this.setupEmbedder()
    this.transactionManager = new TransactionManager()

    // Initialize ready Promise
    // This allows consumers to await brain.ready before using the database
    this._readyPromise = new Promise<void>((resolve, reject) => {
      this._readyResolve = resolve
      this._readyReject = reject
    })
    // Attach a default no-op rejection handler so that init-failure cases
    // (e.g. another writer holds the lock) do not surface as Node
    // "unhandled promise rejection" warnings when callers don't `await brain.ready`.
    // Callers who DO await it still see the original rejection.
    this._readyPromise.catch(() => { /* observed by ready() consumers, if any */ })

    // Track this instance for shutdown hooks
    Brainy.instances.push(this)

    // Index and storage are initialized in init() because they may need each other
  }

  /**
   * Open a Brainy store in read-only mode and initialize it.
   *
   * Convenience factory equivalent to
   * `new Brainy({ ...config, mode: 'reader' })` followed by `init()`. The
   * resulting instance:
   *
   * - Does NOT acquire the writer lock — coexists with a live writer process
   *   and with any number of other readers on the same data directory.
   * - Throws `Cannot mutate a read-only Brainy instance` from every mutation
   *   method (`add`, `addMany`, `update`, `remove`, `removeMany`, `relate`,
   *   `unrelate`, `transact`, `restore`).
   * - Reflects the state of the writer's last successful `flush()`. To force
   *   the writer to flush before opening, call `requestFlush()` on a separate
   *   handle first, or pass `--fresh` to the `brainy inspect` CLI.
   *
   * Typical use cases:
   * - Operator diagnostics during incidents (`brainy inspect` uses this).
   * - Read-replica processes on the same machine.
   * - Long-running analytics scripts that should not contend with the writer.
   *
   * @param config Same options as `new Brainy(config)`. The `mode` field is
   *   ignored and forced to `'reader'`.
   * @returns An initialized, read-only Brainy instance.
   *
   * @example
   * ```typescript
   * const reader = await Brainy.openReadOnly({
   *   storage: { type: 'filesystem', path: '/data/brainy-data/tenant' }
   * })
   * const bookings = await reader.find({ where: { entityType: 'booking' } })
   * await reader.close()
   * ```
   */
  static async openReadOnly<T = any>(config: BrainyConfig): Promise<Brainy<T>> {
    const brain = new Brainy<T>({ ...config, mode: 'reader' })
    await brain.init()
    return brain
  }

  /**
   * Whether this instance is read-only (set via `mode: 'reader'`,
   * `Brainy.openReadOnly()`, or `asOf()`). When true, all mutation methods
   * throw.
   */
  get isReadOnly(): boolean {
    return !this.operationalMode.canWrite
  }

  /**
   * Whether the active storage adapter (anywhere in its prototype chain)
   * implements a given optional method.
   *
   * Storage-adapter plugins (e.g. `@soulcraft/cor`'s `MmapFileSystemStorage
   * extends FileSystemStorage`) inherit new methods Brainy adds to
   * `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the
   * prototype chain, so there's no in-package version skew to worry about as
   * long as the plugin's own dist resolves `@soulcraftlabs/brainy` dynamically
   * (which Cortex 2.2.x onward does — see
   * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`).
   *
   * This helper exists for the **build/install** failure modes the import
   * resolution can't catch:
   *   - Stale `node_modules` left over from a prior `bun install` against
   *     `@soulcraftlabs/brainy ≤7.20.x`.
   *   - Lockfile drift pinning brainy below the version that introduced the
   *     method.
   *   - Docker layer caches that reuse a `node_modules` from an earlier image.
   *   - Bundlers (esbuild, webpack) that freeze the prototype chain at build
   *     time and lose later prototype mutations.
   * In any of those, calling the new method unconditionally crashes boot with
   * `TypeError: storage.X is not a function`. The guard turns that into a
   * loud warning + graceful degradation; the operator's clue is the warning
   * naming the adapter class so they can re-run install / rebuild the image.
   *
   * Storage-adapter authors: see `docs/concepts/storage-adapters.md` for the
   * inheritance contract. Filesystem-backed adapters extending
   * `FileSystemStorage` inherit the 6 multi-process helpers for free; just
   * override `supportsMultiProcessLocking()` → `true` to activate enforcement.
   */
  private hasStorageMethod(name: string): boolean {
    return !!this.storage && typeof (this.storage as unknown as Record<string, unknown>)[name] === 'function'
  }

  /**
   * Throw if this instance cannot perform writes. Called at the top of every
   * mutation method. The check is cheap (a property lookup + boolean test) and
   * runs before any work, so callers get a clear, fast failure in read-only mode.
   */
  private assertWritable(method: string): void {
    if (!this.operationalMode.canWrite) {
      throw new Error(
        `Cannot call ${method}() on a read-only Brainy instance. ` +
        `This instance was opened with mode: 'reader' (or via Brainy.openReadOnly() / asOf()). ` +
        `Open in writer mode to modify data.`
      )
    }
    // Write-quarantine: a prior transaction's rollback failed and left the store
    // inconsistent. Refuse further writes (which would compound the damage) until
    // repairIndex() reconciles and lifts the quarantine. Reads still work.
    if (this.storeInconsistency) {
      throw new Error(
        `Cannot call ${method}() — the store is WRITE-QUARANTINED after a failed ` +
        `transaction rollback left it inconsistent. Reads still work; run repairIndex() ` +
        `to reconcile the derived indexes against canonical storage and lift the ` +
        `quarantine. Original inconsistency: ${this.storeInconsistency.message}`
      )
    }
  }

  /**
   * Initialize Brainy.
   *
   * **Calling this explicitly is optional.** Every public operation lazily
   * initializes the instance on first use (`new Brainy()` followed directly
   * by `add()` / `find()` / `transact()` just works). Call `init()` yourself
   * when you want to control *when* the startup cost is paid (e.g. during
   * server boot rather than on the first request) or to pass init-time
   * configuration overrides.
   *
   * Idempotent and concurrency-safe: repeat calls after success return
   * immediately; calls racing an in-flight initialization await that single
   * run (overrides are only applied by the run that starts initialization).
   *
   * @param overrides Optional configuration overrides for init
   */
  async init(overrides?: Partial<BrainyConfig & { dimensions?: number }>): Promise<void> {
    if (this.initialized) {
      return
    }
    if (this.initPromise) {
      return this.initPromise
    }
    const run = this.performInit(overrides)
    this.initPromise = run
    try {
      await run
    } finally {
      this.initPromise = null
    }
  }

  /**
   * The single initialization run guarded by `init()`. Never call directly —
   * always go through `init()` (or any public method, which lazily inits).
   */
  private async performInit(overrides?: Partial<BrainyConfig & { dimensions?: number }>): Promise<void> {

    // Apply any init-time configuration overrides
    if (overrides) {
      const { dimensions, ...configOverrides } = overrides
      // Storage configs shallow-merge; a pre-constructed adapter instance
      // (on either side) is taken whole — spreading an instance would strip
      // its prototype methods.
      const baseStorage = this.config.storage
      const overrideStorage = configOverrides.storage
      const mergedStorage =
        isStorageAdapterInstance(baseStorage) || isStorageAdapterInstance(overrideStorage)
          ? (overrideStorage ?? baseStorage)
          : { ...baseStorage, ...overrideStorage }
      this.config = {
        ...this.config,
        ...configOverrides,
        storage: mergedStorage,
        verbose: configOverrides.verbose ?? this.config.verbose,
        silent: configOverrides.silent ?? this.config.silent
      }

      // Set dimensions if provided
      if (dimensions) {
        this.dimensions = dimensions
      }

      // Re-derive operationalMode if mode override changed it
      if (configOverrides.mode) {
        this.operationalMode = configOverrides.mode === 'reader'
          ? new ReaderMode()
          : new HybridMode()
      }
    }

    // Configure logging based on config options
    if (this.config.silent) {
      // Store original console methods for restoration
      this.originalConsole = {
        log: console.log,
        info: console.info,
        warn: console.warn,
        error: console.error
      }

      // Override all console methods to completely silence output
      console.log = () => {}
      console.info = () => {}
      console.warn = () => {}
      console.error = () => {}

      // Also configure logger for silent mode
      configureLogger({ level: LogLevel.SILENT })  // Suppress all logs
    } else if (this.config.verbose) {
      configureLogger({ level: LogLevel.DEBUG })  // Enable verbose logging
    }

    // OPEN-PATH NARRATION: phase timing across the five named stretches of
    // init — storage init / generation-store open+fold / index init+gate /
    // VFS bootstrap / embedding-warm-started. Each `markPhase()` call records
    // elapsed ms SINCE THE PREVIOUS checkpoint, so the buckets always sum to
    // the pre-integration/warmOnOpen total.
    //
    // THE LAW THIS ENFORCES: an open is never silent for more than
    // OPEN_HEARTBEAT_MS. A production service opening a 16 GB store logged
    // NOTHING for three minutes and then began work — the operator could not
    // tell a slow open from a hung one, and restarted into the same wall.
    // Two mechanisms, both on the always-visible narration channel (the old
    // breakdown used `prodLog.warn`, which production clamps away — that is
    // why the three minutes were silent):
    //   - a heartbeat that names the phase currently running and its elapsed
    //     wall, every OPEN_HEARTBEAT_MS, for as long as the open lasts;
    //   - one line per phase AS IT ENDS, naming its wall and its cause, for
    //     any phase over OPEN_PHASE_NARRATE_MS.
    // The heartbeat is unref'd and cleared in the `finally` below, so it can
    // neither hold the process open nor outlive a failed init. It cannot fire
    // inside a phase that blocks the event loop synchronously; such a phase
    // must narrate its own progress (the generation-log fold does).
    const OPEN_HEARTBEAT_MS = 5_000
    const OPEN_PHASE_NARRATE_MS = 2_000
    /** Phase order + what each one is paying for, quoted in its narration. */
    const OPEN_PHASES: ReadonlyArray<{ name: string; cause: string }> = [
      { name: 'storage-init', cause: 'opening the store and loading its count ledger' },
      {
        name: 'generation-store-open-fold',
        cause: 'opening the generation store: crash-recovery replay/fold, derived-family registration, format handshake'
      },
      { name: 'index-init-gate', cause: 'constructing the derived indexes and gating them for serving' },
      { name: 'vfs-bootstrap', cause: 'bootstrapping the virtual filesystem' },
      { name: 'embedding-warm-started', cause: 'starting the background embedding warm' }
    ]
    const initStart = Date.now()
    let lastPhaseCheckpoint = initStart
    let currentPhaseIndex = 0
    const phaseTimingsMs: Record<string, number> = {}
    const openHeartbeat: ReturnType<typeof setInterval> = setInterval(() => {
      const phase = OPEN_PHASES[currentPhaseIndex]
      if (!phase) return
      prodLog.narrate(
        `[Brainy] open: still in phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` +
          `"${phase.name}" after ${Math.round((Date.now() - lastPhaseCheckpoint) / 1000)}s ` +
          `(${Math.round((Date.now() - initStart) / 1000)}s into the open) — ${phase.cause}`
      )
    }, OPEN_HEARTBEAT_MS)
    if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref()
    /**
     * Narrate one STEP inside a phase when it turns out to be expensive.
     * A phase that costs a minute and names only itself tells an operator
     * where to look but not what to look at; this names the step. Silent
     * under OPEN_PHASE_NARRATE_MS, so a fast open says nothing extra.
     */
    const step = async <T>(name: string, cause: string, run: () => Promise<T>): Promise<T> => {
      const startedAt = Date.now()
      try {
        return await run()
      } finally {
        const elapsed = Date.now() - startedAt
        if (elapsed >= OPEN_PHASE_NARRATE_MS) {
          prodLog.narrate(`[Brainy] open: step "${name}" took ${elapsed}ms — ${cause}`)
        }
      }
    }
    const markPhase = (name: string): void => {
      const now = Date.now()
      const elapsed = now - lastPhaseCheckpoint
      phaseTimingsMs[name] = elapsed
      lastPhaseCheckpoint = now
      const finished = OPEN_PHASES[currentPhaseIndex]
      if (elapsed >= OPEN_PHASE_NARRATE_MS && finished && finished.name === name) {
        prodLog.narrate(
          `[Brainy] open: phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` +
            `"${name}" finished in ${elapsed}ms — ${finished.cause}`
        )
      }
      currentPhaseIndex++
    }

    try {
      // Auto-detect and activate plugins BEFORE storage setup
      // so plugin-provided storage factories (e.g., filesystem override from cor) are available
      await this.loadPlugins()

      // 7.x → 8.0 layout migration, BEFORE storage setup: collapse a legacy
      // branch layout to the flat 8.0 layout on a built-in FileSystemStorage, so
      // setupStorage() (and any native plugin factory's own legacy guard) sees a
      // clean store. No-op for non-filesystem stores and already-flat brains.
      await this.legacyLayoutMigrationPhase()

      // Setup and initialize storage (checks plugin storage factories first)
      this.storage = await this.setupStorage()
      await this.storage.init()

      // OS-limit detection (once per process, Linux-only, measurement-only):
      // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite
      // at pool scale, instead of letting the operator meet them as EMFILE or
      // a failed mmap deep inside an index open. Fire-and-forget — the check
      // never affects open.
      void warnOnLowOsLimits()

      // Acquire the writer lock for filesystem (and other locking-capable) backends.
      // Skipped in reader mode and on backends that don't support multi-process locking.
      // Throws if another live writer holds the directory (unless force: true).
      //
      // Defensive call: older storage adapters (e.g. `@soulcraft/cor@2.2.0`
      // and earlier) bundle a pre-7.21 `BaseStorage` that doesn't define these
      // methods. We feature-detect each one rather than fail boot.
      if (this.config.mode !== 'reader') {
        const canLock = this.hasStorageMethod('supportsMultiProcessLocking') &&
                        this.storage.supportsMultiProcessLocking()
        if (canLock && this.hasStorageMethod('acquireWriterLock')) {
          await this.storage.acquireWriterLock({ force: this.config.force })
          if (this.hasStorageMethod('startFlushRequestWatcher')) {
            this.storage.startFlushRequestWatcher(async () => {
              if (this.initialized) {
                await this.flush()
              }
            })
          }
        } else if (!this.config.silent) {
          // Older adapter OR a backend that doesn't enforce locking (cloud / memory).
          // Surface this so operators know the multi-process protections aren't active.
          const backendName = this.storage.constructor.name || 'storage'
          if (backendName === 'MemoryStorage') {
            // Memory is single-process by construction — no warning needed.
          } else if (!this.hasStorageMethod('supportsMultiProcessLocking')) {
            // The multi-process methods are inherited from FileSystemStorage
            // when an adapter extends it. Reaching this branch means the
            // prototype chain doesn't resolve to a Brainy version that
            // defines them — almost always a build/install artifact rather
            // than the plugin lacking the code. Tell the operator what to
            // try first.
            console.warn(
              `[brainy] Storage adapter \`${backendName}\` is missing the ` +
              `multi-process methods on its prototype chain. Writer locking ` +
              `and the flush-request RPC are disabled for this directory. ` +
              `Likely fix: clean install (\`rm -rf node_modules bun.lockb && ` +
              `bun install\`) or rebuild your container image to refresh ` +
              `\`@soulcraftlabs/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.`
            )
          } else {
            console.warn(
              `[brainy] Multi-process writer protection is not enforced on ${backendName}. ` +
              `See docs/concepts/multi-process.md for the model.`
            )
          }
        }
      }

      // PHASE 1 of 5 — "storage init": plugin/legacy-layout bootstrap,
      // storage adapter construction+init, the OS-limit check, and the
      // writer-lock claim, all folded into one bucket (everything above this
      // line since performInit started).
      markPhase('storage-init')

      // 8.0 generational MVCC: open the record layer BEFORE any index is
      // created or loaded. Crash recovery may rewrite canonical entity files
      // (restoring before-images of an uncommitted transaction), and every
      // index below loads from those files — opening the store first
      // guarantees indexes never observe rolled-back state. Reader-mode
      // instances skip recovery (readers never write; the next writer
      // repairs).
      this.generationStore = new GenerationStore(this.storage)
      const generationOpenResult = await step(
        'generation-store.open',
        'reading the generation manifest and committed ranges, opening the fact log and the ' +
          'packed segment tier, and folding any crash-recovery replay',
        () => this.generationStore.open({ readOnly: this.config.mode === 'reader' })
      )

      // The generation fact log is CANONICAL state, not a derived index — no
      // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare
      // its namespace as a protected family (rebuildable: false — a lost fact
      // segment is NOT reconstructable) so the storage layer REFUSES such
      // deletes; refusal beats trust. Feature-detected + idempotent per name.
      if (
        this.config.mode !== 'reader' &&
        this.generationStore.getFactLog() &&
        typeof this.storage.registerDerivedFamily === 'function'
      ) {
        await this.storage.registerDerivedFamily({
          name: 'generation-facts',
          members: ['_generations/facts/'],
          namespace: true,
          rebuildable: false
        })
      }

      // Fact-scan capability: wire the storage seam through which index
      // providers (which hold only `storage`) reach the fact log. A closure
      // over the LIVE log — restore/reopen swaps the instance transparently —
      // so a provider's heal can switch from the enumeration walk to one
      // sequential fact scan whenever the log exists.
      if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') {
        ;(this.storage as BaseStorage).setFactScanSource({
          factLog: () => this.generationStore?.getFactLog() ?? null,
          // The committed watermark, exposed as a capability so providers
          // never parse the store's private manifest format.
          committedGeneration: () => this.generationStore?.committedGeneration() ?? 0
        })
      }

      // Entity-tree stamp coherence: compare the stamped sourceGeneration +
      // rollup invariants against the log head + live counters. Loud on
      // genuine incoherence (repairIndex heals), silent on absent/coherent,
      // benign-behind refreshes at the next flush. Never blocks open.
      await step(
        'verify-entity-tree-stamp',
        'comparing the entity tree\'s stamped generation and rollups against the store',
        () => this.verifyEntityTreeStamp()
      )

      // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
      // marker (`_system/brain-format.json`) into an in-memory field NOW —
      // after the store-open phase, but BEFORE any derived index or native
      // provider is constructed below — so `formatInfo()` is populated when the
      // provider reads it synchronously at its own init(). A drifted or absent
      // `indexEpoch` means the on-disk derived-index format predates this build
      // (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds
      // them from the canonical records and then re-stamps the marker AFTER the
      // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old /
      // absent marker, so the next open idempotently re-rebuilds).
      this._brainFormat = await step(
        'read-brain-format',
        'reading the on-disk format marker that decides whether the derived indexes are stale',
        () => readBrainFormat(this.storage)
      )
      this._indexEpochStale =
        this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH

      // Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the
      // on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below.
      // Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a
      // derived index — so a migration bug can be rolled back. Removed once the
      // upgrade verifies + stamps; retained on failure. No-op for a reader, for
      // non-filesystem storage, or for a brain with no persisted data.
      if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) {
        await step(
          'pre-upgrade-backup',
          'snapshotting the brain directory before a one-time format rebuild (migrationBackup)',
          () => this.createMigrationBackupIfNeeded()
        )
      }

      // PHASE 2 of 5 — "generation-store open+fold": GenerationStore
      // construction+open (crash-recovery replay/rollback fold), the
      // derived-family registration, the fact-scan seam, the entity-tree
      // stamp check, the brain-format handshake, and the pre-upgrade backup.
      markPhase('generation-store-open-fold')

      // Provider: embeddings (reassign embedder if plugin provides one)
      const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
      if (embeddingProvider) {
        this.embedder = embeddingProvider
      }

      // Provider: cache (replace global singleton before any consumer uses it)
      const cacheProvider = this.pluginRegistry.getProvider<UnifiedCache>('cache')
      if (cacheProvider) {
        setGlobalCache(cacheProvider)
      }

      // Provider: roaring bitmaps (native CRoaring replacement for WASM)
      const roaringProvider = this.pluginRegistry.getProvider<any>('roaring')
      if (roaringProvider) {
        const { setRoaringImplementation } = await import('./utils/roaring/index.js')
        setRoaringImplementation(roaringProvider)
      }

      // Provider: msgpack (native replacement for JS @msgpack/msgpack)
      const msgpackProvider = this.pluginRegistry.getProvider<any>('msgpack')
      if (msgpackProvider) {
        const { setMsgpackImplementation } = await import('./graph/lsm/SSTable.js')
        setMsgpackImplementation(msgpackProvider)
      }

      // Provider: sort:topK (e.g. cor's native partial-sort / heap-select) — swaps the
      // JS result-ranking used by find() to pick the top `offset + limit` rows. The provider
      // returns indices into a scores array, ordered descending with stable ties, identical
      // to the JS sortTopKIndicesJs. rankIndicesByScore validates the provider's output and
      // falls back to JS on any inconsistency, so ranking is always correct.
      const sortTopKProvider = this.pluginRegistry.getProvider<
        (scores: number[], k: number, descending: boolean) => number[]
      >('sort:topK')
      if (sortTopKProvider) {
        const { setSortTopKImplementation } = await import('./utils/resultRanking.js')
        setSortTopKImplementation(sortTopKProvider)
      }

      // Provider: distance function (resolve BEFORE setupIndex — index uses this.distance)
      const nativeDistance = this.pluginRegistry.getProvider<DistanceFunction>('distance')
      if (nativeDistance) {
        this.distance = nativeDistance
      }

      // Provider: HNSW index factory (plugin or JS fallback)
      this.index = this.createIndex()

      // Provider: metadata index factory
      const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
      if (metadataFactory) {
        this.metadataIndex = metadataFactory(this.storage)
      } else {
        // JS fallback — inject native EntityIdMapper if cor provides one
        const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper')
        this.metadataIndex = new MetadataIndexManager(this.storage, {}, {
          entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined,
        })
      }

      // Provider: graph index factory
      const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
      if (graphFactory) {
        this.graphIndex = graphFactory(this.storage)
        this.storage.setGraphIndex(this.graphIndex)
        await this.metadataIndex.init()
      } else {
        const [, graphIndex] = await Promise.all([
          this.metadataIndex.init(),
          this.storage.getGraphIndex()
        ])
        this.graphIndex = graphIndex
      }

      // Fact-log v2 mint seam: after-image records carry minted dense ints,
      // and the ONE authority for those assignments is the metadata index's
      // id mapper (append-only getOrAssign — a rebuilt mapper reproduces
      // them exactly). The generation store cannot know the mapper, so the
      // mint thunk is injected here, immediately after the index is ready;
      // installing it is what flips the fact log's LIVE writes to the v2
      // segment format. A configuration whose mapper is unavailable throws
      // at mint time — an int of 0 is never written.
      this.generationStore.setIntMinter((kind, id) => {
        const mapper = this.metadataIndex?.getIdMapper?.()
        if (!mapper || typeof mapper.getOrAssign !== 'function') {
          throw new Error(
            `fact log v2: cannot mint the ${kind} int for ${id} — the metadata index's ` +
              `id mapper is unavailable on this configuration; refusing to write an ` +
              `after-image without a reproducible int`
          )
        }
        const minted = mapper.getOrAssign(id, undefined)
        const asBigint = typeof minted === 'bigint' ? minted : BigInt(minted)
        // THE RESERVED-ROOT EXEMPTION: the VFS root (the all-zeros UUID) is
        // minted int 0 BY CONSTRUCTION at genesis on existing brains — the
        // one legitimate zero in the id space. Zero for ANY other id is a
        // corrupt mint and refuses. (Without this, every existing brain's
        // adoption oracle false-flagged its own root and refused the flip.)
        const isReservedRoot =
          asBigint === 0n && id === '00000000-0000-0000-0000-000000000000'
        if (asBigint < 0n || (asBigint === 0n && !isReservedRoot)) {
          throw new Error(
            `fact log v2: the id mapper minted ${asBigint} for ${kind} ${id} — ` +
              `minted ints are positive (int 0 is reserved for the VFS root alone); ` +
              `refusing to write`
          )
        }
        return asBigint
      })

      // Eager cold-load (readiness contract). A provider that persists its
      // derived state exposes init?(): trigger the load NOW — AFTER
      // metadataIndex.init() above (the id-mapper is hydrated first, so a
      // native int-keyed index resolves endpoints/slots through it — the
      // CTX-BR-RESTORE-REBUILD order) and BEFORE the rebuild gate — so a
      // durable index reports its real size()/isReady() at the gate instead
      // of eating a spurious rebuild-from-canonical on every open (§7.1
      // rebuild()==0). Vector first, then graph. JS engines: the JS vector
      // index has no init() (rebuild() IS its load path); the JS graph's
      // init() already cold-loaded its LSM inside storage.getGraphIndex()
      // (idempotent here).
      const vectorWithInit = this.index as { init?: () => Promise<void> }
      if (typeof vectorWithInit.init === 'function') {
        await vectorWithInit.init()
      }
      const graphWithInit = this.graphIndex as { init?: () => Promise<void> }
      if (typeof graphWithInit.init === 'function') {
        await graphWithInit.init()
      }

      // 8.0 u64 contract: thread the shared UUID ↔ int resolver into the JS
      // graph index and the storage layer's verb read paths.
      this.wireGraphIdResolver()

      // Wire the connections codec (2.4.0 #3). When the graph:compression
      // provider is registered AND the metadata index exposes a stable
      // idMapper, inject a codec that encodes JS HNSW connections as
      // delta-varint blobs at save time and decodes on load. It rides the
      // storage adapter's generic binary-blob primitive (both the filesystem
      // and memory adapters implement it), so it is independent of any native
      // vector provider's single-file on-disk format.
      this.wireConnectionsCodec()

      // 8.0 generational MVCC: if crash recovery rolled back an uncommitted
      // transaction, every derived index is suspect — persisted index state
      // (flushed before the crash) may reference the rolled-back writes.
      // Rebuild all three from the repaired canonical records (the JS-index
      // equivalent of the locked design's "manifest-vs-index generation
      // comparison + replay on open").
      if (generationOpenResult.rolledBackGenerations > 0) {
        prodLog.warn(
          `[Brainy] Rebuilding indexes after crash recovery rolled back ` +
            `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)`
        )
        // SELF-REBUILD DEFERENCE, same law as the open gate: a provider that
        // is already rebuilding itself from canonical is doing exactly this
        // work. Kicking a second rebuild on top of it is redundant at best.
        // Safe by ordering: the crash-recovery fold ran in the generation
        // store's open, BEFORE any provider was constructed, so a provider
        // rebuilding now is reading the repaired canonical records.
        const kick = async (leg: string, provider: { rebuild: () => Promise<void> }) => {
          const rebuilding = assessProviderRebuild(provider)
          if (rebuilding) {
            prodLog.narrate(
              `[Brainy] crash-recovery rebuild: the ${leg} provider is already ` +
                `${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.`
            )
            return
          }
          await provider.rebuild()
        }
        await Promise.all([
          kick('metadata', this.metadataIndex),
          kick('vector', this.index as unknown as { rebuild: () => Promise<void> }),
          kick('graph', this.graphIndex)
        ])
      }

      // METADATA WATERMARK CATCHUP: the JS metadata index computed its
      // three-way watermark verdict inside metadataIndex.init() above,
      // against the generation store's now-FINAL committed generation (the
      // crash-recovery fold above — the durable-at-ack replay of acked
      // writes whose canonical bytes hadn't reached disk — has already run,
      // and any rolled-back-transaction rebuild just above already brought
      // every index current, so the verdict is consumed here whether or not
      // that rebuild ran). Consumed BEFORE the rebuild gate below and BEFORE
      // this open serves any read — the cure for the class of bug where
      // canonical get()/counts recover a crash-window write but find()
      // keeps serving the metadata index's pre-crash state (the index
      // flushes only periodically, not per-commit).
      await this.consumeMetadataWatermarkVerdict(generationOpenResult.rolledBackGenerations > 0)

      // 8.0 versioned-provider replay-gap check: a provider whose persisted
      // index generation is behind the storage layer's committed generation
      // replays the gap itself (post-commit applier contract) — surface the
      // gap for observability.
      for (const provider of this.versionedIndexProviders()) {
        const providerGen = provider.generation()
        // Defensive finite-integer guard: committedGeneration() is validated
        // at the store's open (torn artifacts discard, narrated) — but a
        // RangeError here would kill the whole open, so the consumer guards
        // too. A non-finite value narrates and skips the gap check (the
        // provider's own replay contract still governs).
        const committedRaw = this.generationStore.committedGeneration()
        if (!Number.isSafeInteger(committedRaw) || committedRaw < 0) {
          prodLog.warn(
            `[Brainy] committed generation is non-integer (${String(committedRaw)}) at ` +
              `init — torn-artifact survivor; skipping the provider replay-gap check`
          )
          continue
        }
        const committed = BigInt(committedRaw)
        if (providerGen < committed) {
          prodLog.info(
            `[Brainy] Versioned index provider is at generation ${providerGen} ` +
              `(storage committed: ${committed}) — provider replays the gap per ` +
              `the post-commit applier contract`
          )
        } else if (providerGen > committed) {
          // The AHEAD direction is incoherence, not a replay gap: the provider's
          // persisted index claims writes the store no longer has — the signature
          // of a torn copy or a log truncation that pulled the committed
          // watermark back (crash recovery, byte-copy of a live store). A replay
          // can never converge on it and index answers may reference vanished
          // writes. Name it loudly at open so it is never diagnosed from a
          // silent journal; the provider's own coherence check / heal walk (or
          // brain.repairIndex()) is the cure.
          prodLog.warn(
            `[Brainy] Versioned index provider is AHEAD of the store: provider ` +
              `generation ${providerGen} vs committed ${committed}. This store was ` +
              `likely copied from a live service or truncated during crash recovery. ` +
              `Derived-index answers may reference rolled-back writes until the ` +
              `provider heals from canonical (brain.repairIndex() forces it).`
          )
        }
      }

      // Recover VFS content blobs a 7→8 upgrade left stranded in the removed
      // branch system's copy-on-write area (`_cow/`). Runs BEFORE the rebuild
      // (whose success stamp removes the pre-upgrade backup) so the backup still
      // covers this step, and works on the raw object primitives (no blob store
      // needed yet). A cheap no-op on native-8.0 / fresh brains and on a brain
      // already healed. See adoptLegacyCowBlobs.
      await this.autoAdoptLegacyVfsBlobsIfNeeded()

      // Temporal-blob contract: one-time (marker-gated) backfill of blob
      // history reference counts for stores whose generation history predates
      // the contract — after it, compaction reclaims blob bytes exactly (zero
      // live AND zero history references). On a scrub failure the store runs
      // leak-safe (no blob reclamation) rather than risk a premature delete.
      if (typeof (this.storage as unknown as {
        backfillBlobHistoryRefCountsIfNeeded?: () => Promise<void>
      }).backfillBlobHistoryRefCountsIfNeeded === 'function') {
        await (this.storage as unknown as {
          backfillBlobHistoryRefCountsIfNeeded: () => Promise<void>
        }).backfillBlobHistoryRefCountsIfNeeded()
      }

      // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS
      // root BEFORE the vector-leg open gate below ever compares the
      // canonical vectored-noun count against the vector index's size — see
      // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe
      // O(1) exception to "nothing at open may scale with brain size", and
      // why it must run here rather than waiting on VirtualFileSystem's own
      // (VFS-instance-gated) lazy migration.
      await this.migrateLegacyZeroNormVfsRootIfNeeded()

      // Rebuild indexes if needed for existing data. Runs to completion before
      // init() returns — there is no more first-query lazy path, so the flag
      // below (kept for getIndexStatus() API compatibility) simply flips true
      // once this open-time step has run.
      await step(
        'rebuild-indexes-if-needed',
        'the derived-index gate: each family\'s readiness verdict, and any build it asks for',
        () => this.rebuildIndexesIfNeeded()
      )
      this.lazyRebuildCompleted = true

      // Check for pending data migrations
      await this.checkMigrations()

      // PHASE 3 of 5 — "index init+gate": provider wiring (embeddings,
      // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph
      // index construction, the eager cold-load, id-resolver + connections-
      // codec wiring, crash-recovery index rebuild, the replay-gap check,
      // legacy VFS blob adoption, blob-history backfill, the legacy
      // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate
      // + migration check.
      markPhase('index-init-gate')

      // Register shutdown hooks for graceful count flushing (once globally)
      if (!Brainy.shutdownHooksRegisteredGlobally) {
        this.registerShutdownHooks()
        Brainy.shutdownHooksRegisteredGlobally = true
      }

      // Initialize the content-addressed blob store before VFS — the VFS
      // stores all file content through it.
      if (typeof this.storage.initializeBlobStorage === 'function') {
        await this.storage.initializeBlobStorage()
      }

      // Log provider summary after all wiring is complete
      // Shows developers exactly what's native vs falling back to JS
      if (this.pluginRegistry.hasActivePlugins() && !this.config.silent) {
        const wellKnownKeys = [
          'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache',
          'vector', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
        ]
        const native = wellKnownKeys.filter(k => this.pluginRegistry.hasProvider(k))
        const fallback = wellKnownKeys.filter(k => !this.pluginRegistry.hasProvider(k))
        const plugins = this.pluginRegistry.getActivePlugins().join(', ')
        if (fallback.length === 0) {
          console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins})`)
        } else {
          console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins}) | default: ${fallback.join(', ')}`)
        }
        // An activated accelerator that registered NOTHING is running as pure
        // decoration — every query silently serves from the JS engines. Most
        // often a licensing gate declining to engage. Say so, loudly, so an
        // evaluator never concludes the accelerator "does nothing".
        if (native.length === 0) {
          console.warn(
            `[brainy] ⚠ ${plugins} activated but registered 0 native providers — all queries are ` +
              `running on the default JS engines. Check the plugin's requirements (e.g. a license ` +
              `key) and its logs; see docs/PLUGINS.md.`
          )
        }
      }

      // Mark as initialized BEFORE VFS init
      // VFS.init() needs brain to be marked initialized to call brain methods
      this.initialized = true

      // Migration LOCK (#18): if a native provider is running the one-time
      // 7.x → 8.0 rebuild, WAIT for it to finish HERE — before VFS bootstrap and
      // before the brain serves — so init-internal reads/writes (the VFS root
      // get/add/find below) run against the rebuilt indexes, never a half-built
      // one. This is the "not ready until upgraded" contract: a blocking upgrade
      // to a known-good state. Bounded by `migrationWaitTimeoutMs`: a brain whose
      // upgrade outlives the budget surfaces MigrationInProgressError here (raise
      // the budget, or run the offline migrator) instead of bootstrapping the VFS
      // against a partial index. A non-migrating brain returns instantly.
      await this.awaitMigrationLock()

      // Initialize VFS: Ensure VFS is ready when accessed as property
      // This eliminates need for separate vfs.init() calls - zero additional complexity
      this._vfs = new VirtualFileSystem(this)
      await step(
        'vfs.init',
        'creating or adopting the VFS root and wiring the path resolver',
        () => this._vfs!.init()
      )
      this._vfsInitialized = true  // Mark VFS as fully initialized

      // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the
      // generation-0 baseline. From here, every single-op write is a Model-B
      // generation. (Writers only — readers never stamp; a no-op for them.)
      if (!this.isReadOnly) {
        this._generationStampingActive = true
      }

      // LOG-AUTHORITY SWITCH (checked at open only). A STORED artifact
      // always wins: an already-flipped brain runs durable-at-ack; an
      // explicitly-recorded tree posture is honored. With NO artifact, the
      // 10.0.0 FLEET DEFAULT is ADOPT-AT-OPEN (config logAuthority:
      // 'adopt'): the verification oracle gates the flip — curable
      // divergences are baseline-backfilled, the brain flips ONLY on green,
      // and a brain that cannot go green STAYS tree-authoritative LOUDLY
      // with the refusal recorded (cheap subsequent opens; an operator
      // re-runs adoptLogAuthority() after fixing the divergence).
      // 'defer' is the documented opt-out: no automatic adoption.
      if (!this.isReadOnly) {
        const storedArtifact = await this.storage
          .readRawObject(LOG_AUTHORITY_PATH)
          .catch(() => null)
        const authority = await step(
          'read-log-authority',
          'reading the stored storage-authority artifact',
          () => readLogAuthority(this.storage)
        )
        this._logAuthority = authority
        if (authority.authority === 'log') {
          this.generationStore.setLogDurability('at-ack')
          prodLog.info('[Brainy] storage authority: generation log (durable-at-ack enabled)')
        } else if (
          storedArtifact === null &&
          this.config.logAuthority === 'adopt' &&
          this.generationStore.getFactLog() !== null
        ) {
          try {
            await step(
              'adopt-log-authority',
              'the adoption oracle: verifying the log against canonical before flipping this ' +
                'brain to durable-at-ack, and backfilling any curable divergence',
              () => this.adoptLogAuthority()
            )
            prodLog.info(
              '[Brainy] storage authority adopted at open: generation log ' +
                '(fleet default; oracle green; durable-at-ack enabled)'
            )
          } catch (err) {
            // The guarded ruling: a brain that cannot verify STAYS tree,
            // loudly, with the refusal recorded so subsequent opens are
            // cheap. Never a silent half-state; never a failed open.
            const reason = (err as Error).message
            prodLog.warn(
              `[Brainy] log-authority adoption REFUSED at open — this brain stays ` +
                `tree-authoritative until an operator resolves the divergence and ` +
                `re-runs adoptLogAuthority(). Reason: ${reason}`
            )
            try {
              const refusal: LogAuthorityRecord = {
                authority: 'tree',
                adoptRefusal: { at: Date.now(), reason: reason.slice(0, 500) }
              }
              await this.storage.writeRawObject(LOG_AUTHORITY_PATH, refusal)
              this._logAuthority = refusal
            } catch {
              // Unrecordable refusal = the next open retries the oracle —
              // the conservative outcome.
            }
          }
        }
      }

      // MT5 crash recovery — REPLAY, NOT LISTING: the pending-embed markers
      // live IN the generation log (embed.pending rides the deferred write's
      // own fact; embed.landed rides the landing commit), so recovery folds
      // the log's marker records back into the in-memory set — after the
      // one-time bridge migrates any sidecar files a pre-log build left
      // behind — and resumes the worker in the background. A crash between
      // a deferred write's ack and its background embed DELAYED a vector;
      // this is where it lands.
      if (!this.isReadOnly) {
        try {
          await step(
            'bridge-pending-embed-sidecars',
            'migrating any pre-log deferred-embed marker files into the generation log',
            () => this.bridgeLegacyPendingEmbedSidecars()
          )
          await step(
            'recover-pending-embeds',
            'folding the generation log\'s deferred-embed markers back into the pending set',
            () => this.recoverPendingEmbedsFromLog()
          )
          if (this._pendingEmbedIds.size > 0) {
            prodLog.info(
              `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` +
                `session — resuming in the background`
            )
            const t = setTimeout(() => this.kickEmbedWorker(), 0)
            ;(t as { unref?: () => void }).unref?.()
          }
        } catch (err) {
          prodLog.warn(
            `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` +
              `the log's markers remain durable; recovery retries next open`
          )
        }
      }

      // PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob
      // storage init, the provider-summary log, flipping `initialized`,
      // the migration-lock wait, VFS construction+init, flipping generation
      // stamping active, the log-authority adopt/oracle check, and
      // pending-embed crash recovery.
      markPhase('vfs-bootstrap')

      // Eager embedding initialization — BACKGROUND WARM (open-path fix).
      //
      // Adaptive default (8.0): the WASM embedding engine eagerly WARMS
      // during init() WHENEVER it is the active embedder — i.e. no native
      // 'embeddings' provider has taken over — and the instance is a writer
      // (not reader-mode) outside of unit tests. The WASM module (≈93MB with
      // the embedded model) takes 90-140s to compile on throttled CPUs.
      //
      // Historically this AWAITED `embeddingManager.init()` INLINE, so every
      // writer's open() blocked on the compile — N concurrent opens all
      // queued on the ONE process-global singleton (an ~80x contention
      // multiplier measured in a production restart storm: 90,017ms busy vs
      // 1,117ms quiet). The engine only needs to be ready before the FIRST
      // REAL embed() call, not before init() returns, so this now only
      // STARTS the warm and moves on — init() never waits for it.
      //
      // No double-await needed for correctness: `this.embed()` (~line 15420)
      // delegates to `this.embedder`, which for the default engine is
      // `embeddingManager.getEmbeddingFunction()` → `embeddingManager.embed()`
      // (src/embeddings/EmbeddingManager.ts). That method calls `await
      // this.init()` FIRST, and `init()` itself serializes every concurrent
      // caller onto ONE shared `globalInitPromise` — so the first real
      // embed() automatically waits for whichever finishes first: this
      // background warm (if still running) or a fresh init() (if the warm
      // hasn't reached this code yet, e.g. `eagerEmbeddings: false`).
      // Verified by reading both call sites; `_embeddingWarmPromise` below
      // is stored for observability only, never re-awaited by embed().
      //
      // Skipped automatically when:
      //   - a native 'embeddings' provider is registered (it owns embeddings;
      //     the WASM engine is dead weight),
      //   - reader-mode (readers don't embed — they query existing vectors),
      //   - unit-test mode (tests must stay fast and use the mock embedder).
      //
      // `eagerEmbeddings: false` keeps meaning "no warm at all" — fully lazy,
      // the first embed() call pays the full cost inline, same as before.
      const isUnitTestMode = isDeterministicEmbedMode()
      const eager = this.config.eagerEmbeddings ?? true
      if (
        eager &&
        !this.pluginRegistry.hasProvider('embeddings') &&
        this.config.mode !== 'reader' &&
        !isUnitTestMode
      ) {
        const warmStart = Date.now()
        console.log('Background embedding-engine warm started (init() does not wait for it)...')
        this._embeddingWarmPromise = embeddingManager
          .init()
          .then(() => {
            prodLog.info(
              `[Brainy] background embedding-engine warm complete in ${Date.now() - warmStart}ms`
            )
          })
          .catch((err) => {
            // Loud, never silent: a warm that fails to compile must be
            // heard NOW, not discovered as a mystery latency spike on
            // whichever request happens to trigger the first real embed().
            // That first embed() call still retries init() itself (the
            // singleton promise contract above) and surfaces its own typed
            // error to its caller — this is the immediate, background echo.
            prodLog.warn(
              `[Brainy] background embedding-engine warm FAILED: ` +
                `${(err as Error).message} — the first embed() call will retry ` +
                `initialization and surface the error there`
            )
          })
      }

      // PHASE 5 of 5 — "embedding-warm-started": just the synchronous cost
      // of kicking off the background warm above (the warm's own compile
      // time is NOT included — that's the whole point of backgrounding it).
      markPhase('embedding-warm-started')
      {
        const totalOpenMs = Date.now() - initStart
        if (totalOpenMs > 2000) {
          const phaseList = Object.entries(phaseTimingsMs)
            .map(([name, ms]) => `${name}=${ms}ms`)
            .join(', ')
          prodLog.narrate(
            `[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` +
              `phase breakdown above to find which one to investigate first`
          )
        }
      }

      // Integration Hub initialization
      // Creates the hub when integrations are enabled in config
      // Uses dynamic import for tree-shaking when integrations are disabled
      if (this.config.integrations) {
        const hubConfig = this.config.integrations === true
          ? { enable: 'all' as const }
          : this.config.integrations

        const { IntegrationHub } = await import('./integrations/core/IntegrationHub.js')
        this._hub = await IntegrationHub.create(this, {
          basePath: hubConfig.basePath,
          enable: hubConfig.enable,
          // Boundary: the public `IntegrationsConfig['config']` (docs-facing,
          // src/types/brainy.types.ts) and the hub's internal
          // `IntegrationHubConfig['config']` evolved as separate declarations.
          // The hub passes per-integration overrides through verbatim, so the
          // shapes are runtime-compatible, but TypeScript's weak-type check
          // rejects the bridge (e.g. `webhooks.maxRetries` exists only on the
          // public side).
          config: hubConfig.config as unknown as IntegrationHubConfig['config']
        })
      }

      // Eager index warm (operator opt-in, `warmOnOpen: true`). Runs AFTER
      // every step above — index construction, crash recovery, migrations,
      // VFS bootstrap — never in place of any of it, so `warm()` always
      // operates on a fully-initialized brain. Blocking BY DESIGN: the
      // operator traded a longer startup for a first-request that runs at
      // steady-state cost instead of paying demand-load latency on the
      // critical path. See the `warmOnOpen` JSDoc in brainy.types.ts.
      if (this.config.warmOnOpen) {
        await this.warm()
      }

      // Resolve ready Promise - consumers awaiting brain.ready will now proceed
      if (this._readyResolve) {
        this._readyResolve()
      }
    } catch (error) {
      // Reject ready Promise - consumers awaiting brain.ready will receive error
      if (this._readyReject) {
        this._readyReject(error instanceof Error ? error : new Error(String(error)))
      }
      // Machine-readable init failures pass through UNWRAPPED — the writer-lock
      // conflict documents an err.code/err.lockInfo contract ("callers detect
      // this case via err.code"), and wrapping in a fresh Error silently
      // stripped both, leaving consumers only a message to regex against.
      if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') {
        throw error
      }
      // Wrap with the original as `cause` so the originating frame (a plugin's
      // own file:line, e.g. a provider boot failure) survives to the caller's
      // log — a plain string interpolation discards both stack and cause.
      const message = error instanceof Error ? error.message : String(error)
      throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error })
    } finally {
      // The open is over — succeeded or failed. Stop the heartbeat here so a
      // failed init never leaves a timer narrating a phase nobody is running.
      clearInterval(openHeartbeat)
    }
  }

  /**
   * Register shutdown hooks for graceful count flushing
   *
   * Ensures pending count batches are persisted before container shutdown.
   * Critical for Cloud Run, Fargate, Lambda, and other containerized deployments.
   *
   * Handles:
   * - SIGTERM: Graceful termination (Cloud Run, Fargate, Lambda)
   * - SIGINT: Ctrl+C (development/local testing)
   * - beforeExit: Node.js cleanup hook (fallback)
   *
   * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning
   */
  private registerShutdownHooks(): void {
    /**
     * The signal-path shutdown. THREE LAWS, each written by a production
     * shutdown that looked clean and wasn't:
     *
     * 1. PER-INSTANCE ISOLATION. This used to be one `try` around a loop over
     *    every open brain: the first instance whose flush rejected aborted the
     *    loop, so every remaining brain kept its writer lock and its unwritten
     *    markers — and the process still exited 0. A pool of brains failed in
     *    a batch, not one at a time.
     * 2. THE MARKER IS PART OF SHUTDOWN. Flushing the indexes without closing
     *    the generation store leaves the clean-shutdown marker unwritten, so
     *    the NEXT open reads the store as crashed and folds the whole
     *    generation log — measured in tens of seconds on a real store, paid on
     *    every restart, after a shutdown the operator saw exit 0.
     * 3. THE LOCK IS ALWAYS GIVEN UP. In a `finally`, per instance: a process
     *    on its way out holds nothing.
     */
    const flushOnShutdown = async () => {
      console.log('Shutdown signal received - flushing pending data...')
      let flushedCount = 0
      let failedCount = 0
      // Snapshot: close() splices Brainy.instances while we iterate.
      for (const instance of [...Brainy.instances]) {
        if (!instance.initialized) continue
        try {
          // Flush all buffered data (parallel across components, this brain only).
          await Promise.all([
            (async () => {
              if (instance.storage && typeof instance.storage.flushCounts === 'function') {
                await instance.storage.flushCounts()
              }
            })(),
            (async () => {
              if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') {
                await instance.metadataIndex.flush()
              }
            })(),
            (async () => {
              if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') {
                await instance.graphIndex.flush()
              }
            })(),
            (async () => {
              if (instance.index && typeof instance.index.flush === 'function') {
                await instance.index.flush()
              }
            })()
          ])

          // Close the generation store: persists the counter, advances the
          // fold checkpoint, and stamps the clean-shutdown marker LAST — the
          // one step that decides whether the next open adopts or folds. Law 2.
          if (instance.generationStore && !instance.isReadOnly) {
            await instance.generationStore.close()
          }

          // Close components to stop timers that would prevent clean process exit
          await Promise.all([
            (async () => {
              if (instance.graphIndex && typeof instance.graphIndex.close === 'function') {
                await instance.graphIndex.close()
              }
            })(),
            (async () => {
              const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks
              if (index && typeof index.close === 'function') {
                await index.close()
              }
            })(),
            (async () => {
              const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks
              if (metadataIndex && typeof metadataIndex.close === 'function') {
                await metadataIndex.close()
              }
            })()
          ])
          flushedCount++
        } catch (error) {
          failedCount++
          console.error('Failed to flush one Brainy instance on shutdown:', error)
        } finally {
          // Law 3 — the lock and the watcher go regardless.
          try {
            if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') {
              instance.storage.stopFlushRequestWatcher()
            }
          } catch (error) {
            console.error('Failed to stop the flush-request watcher on shutdown:', error)
          }
          try {
            if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') {
              await instance.storage.releaseWriterLock()
            }
          } catch (error) {
            console.error('Failed to release the writer lock on shutdown:', error)
          }
        }
      }
      if (flushedCount > 0) {
        console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`)
      }
      if (failedCount > 0) {
        console.error(
          `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` +
            `their writer locks were released, but their next open will run crash recovery.`
        )
      }
    }

    // Graceful shutdown signals (registered once globally). The listeners are
    // kept as statics so the last live instance's close() can deregister them
    // — the signal handles they hold are ref'd and would otherwise keep the
    // process alive forever after every brain is closed.
    /**
     * Exit the process ONLY when Brainy is the sole handler for this signal.
     *
     * Registering a signal listener suppresses Node's default terminate
     * behaviour, so a library that attaches one must either exit or be sure
     * someone else will. Brainy attaching one AND exiting was the wrong half
     * of that choice for every host application with its own graceful
     * shutdown: both handlers run concurrently, and whichever finishes first
     * wins — a library flush finishing before an application's close()
     * terminated that close mid-flight, at exit code 0, with locks and
     * markers unwritten. When the host has its own handler (listener count
     * above our own), the host owns the exit; Brainy only makes its data
     * durable and steps aside.
     */
    const exitIfSoleShutdownOwner = (signal: 'SIGTERM' | 'SIGINT'): void => {
      if (process.listenerCount(signal) <= 1) {
        process.exit(0)
      }
    }
    Brainy.sigtermListener = async () => {
      await flushOnShutdown()
      exitIfSoleShutdownOwner('SIGTERM')
    }
    Brainy.sigintListener = async () => {
      await flushOnShutdown()
      exitIfSoleShutdownOwner('SIGINT')
    }
    Brainy.beforeExitListener = async () => {
      // Self-deregister FIRST: Node re-emits 'beforeExit' after every event-
      // loop drain, and this flush schedules new async work — with the
      // listener still attached, a script that never calls close() would spin
      // flush → drain → flush forever and never exit. One flush, then the
      // next drain finds no listener and the process exits.
      if (Brainy.beforeExitListener) {
        process.off('beforeExit', Brainy.beforeExitListener)
        Brainy.beforeExitListener = undefined
      }
      await flushOnShutdown()
    }
    process.on('SIGTERM', Brainy.sigtermListener)
    process.on('SIGINT', Brainy.sigintListener)
    process.on('beforeExit', Brainy.beforeExitListener)
  }

  /**
   * Deregister the global shutdown hooks once NO live instance remains, so a
   * script that closed every brain exits on its own — a library must never
   * keep its host process alive. Re-initializing later re-registers them
   * (the `shutdownHooksRegisteredGlobally` flag resets here).
   */
  private static deregisterShutdownHooksIfIdle(): void {
    if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally) {
      return
    }
    if (Brainy.sigtermListener) process.off('SIGTERM', Brainy.sigtermListener)
    if (Brainy.sigintListener) process.off('SIGINT', Brainy.sigintListener)
    if (Brainy.beforeExitListener) process.off('beforeExit', Brainy.beforeExitListener)
    Brainy.sigtermListener = undefined
    Brainy.sigintListener = undefined
    Brainy.beforeExitListener = undefined
    Brainy.shutdownHooksRegisteredGlobally = false
  }

  /**
   * Ensure Brainy is initialized, lazily initializing on first use.
   *
   * Called at the top of every public operation, so `new Brainy()` followed
   * directly by `add()` / `find()` / any other call just works — forgetting
   * `init()` is not an error. Concurrent first operations share a single
   * initialization run (see `init()`); if initialization fails, the error
   * propagates to the operation that triggered it.
   *
   * `close()` is terminal: a closed instance throws here instead of silently
   * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init
   * opportunity.
   */
  private async ensureInitialized(opts?: {
    bypassMigrationLock?: boolean
    needs?: IndexFamily[]
  }): Promise<void> {
    if (this.closed) {
      throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.')
    }
    if (!this.initialized) {
      await this.init()
    }
    // Coordinated migration LOCK (#18): every data-plane read and write funnels
    // through here, so this is the single choke point that holds operations while
    // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no
    // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the
    // derived-index families this operation actually consults, so a read served
    // entirely from canonical storage (`needs: []`) or from a healthy family
    // never blocks on an UNRELATED family's migration. `needs` omitted = the
    // conservative whole-brain wait (writes, and any read not yet classified).
    // Observability (`health`/`checkHealth`) and the lock-clearing path
    // (`stampBrainFormat`, which does not route through here) opt out entirely so
    // an operator can always watch progress and a native provider can stamp.
    // A brain that never migrates pays one boolean check (see awaitMigrationLock).
    if (!opts?.bypassMigrationLock) {
      await this.awaitMigrationLock(opts?.needs)
    }
  }

  /**
   * Check if Brainy is initialized
   */
  get isInitialized(): boolean {
    return this.initialized
  }

  /**
   * Promise that resolves when Brainy is fully initialized and ready to use
   *
   * This Promise is created in the constructor and resolves when init() completes.
   * It can be awaited multiple times safely - the result is cached.
   *
   * This enables reliable readiness detection for consumers — await it
   * before issuing queries when init() was fired without awaiting.
   *
   * @example Waiting for readiness before API calls
   * ```typescript
   * const brain = new Brainy()
   * brain.init() // Fire and forget
   *
   * // Elsewhere in your code (e.g., API handler)
   * await brain.ready
   * const results = await brain.find({ query: 'test' })
   * ```
   *
   * @example Server startup pattern
   * ```typescript
   * const brain = new Brainy()
   * await brain.init()
   *
   * // For health check endpoint
   * app.get('/health', async (req, res) => {
   *   try {
   *     await brain.ready
   *     res.json({ status: 'ready' })
   *   } catch (error) {
   *     res.status(503).json({ status: 'initializing', error: error.message })
   *   }
   * })
   * ```
   *
   * @returns Promise that resolves when init() completes, or rejects if init fails
   */
  get ready(): Promise<void> {
    if (!this._readyPromise) {
      // This should never happen if constructor ran, but handle gracefully
      return Promise.reject(new Error('Brainy not constructed properly'))
    }
    return this._readyPromise
  }

  // ============= CORE CRUD OPERATIONS =============

  /**
   * Add an entity to the database
   *
   * **Data vs Metadata:**
   * - `data`: Content used for vector embeddings. Searchable via **semantic similarity**
   *   (HNSW vector index). NOT queryable via `where` filters. Pass a string for text
   *   embedding, or any value for opaque storage.
   * - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where`
   *   filters in `find()`. Put anything you want to filter/query on here (tags,
   *   categories, dates, flags, etc.).
   *
   * @param params - Parameters for adding the entity
   * @param params.data - Content to embed and store (required). Strings are auto-embedded.
   * @param params.type - NounType classification (required)
   * @param params.metadata - Custom queryable metadata (indexed, used in where filters)
   * @param params.id - Custom ID (auto-generated UUID if not provided)
   * @param params.vector - Pre-computed embedding vector (skips auto-embedding)
   * @param params.service - Service name for multi-tenancy
   * @param params.confidence - Type classification confidence (0-1)
   * @param params.weight - Entity importance/salience (0-1)
   * @returns Promise that resolves to the entity ID
   *
   * @example Basic entity creation
   * ```typescript
   * const id = await brain.add({
   *   data: "John Smith is a software engineer",
   *   type: NounType.Person,
   *   metadata: { role: "engineer", team: "backend" }
   * })
   * console.log(`Created entity: ${id}`)
   * ```
   *
   * @example Adding with confidence and weight
   * ```typescript
   * const id = await brain.add({
   *   data: "Machine learning model for sentiment analysis",
   *   type: NounType.Concept,
   *   metadata: { accuracy: 0.95, version: "2.1" },
   *   confidence: 0.92,  // High confidence in Concept classification
   *   weight: 0.85       // High importance entity
   * })
   * ```
   *
   * @example Adding with custom ID
   * ```typescript
   * const customId = await brain.add({
   *   id: "user-12345",
   *   data: "Important document content",
   *   type: NounType.Document,
   *   metadata: { priority: "high", department: "legal" }
   * })
   * ```
   *
   * @example Using pre-computed vector (optimization)
   * ```typescript
   * const vector = await brain.embed("Optimized content")
   * const id = await brain.add({
   *   data: "Optimized content",
   *   type: NounType.Document,
   *   vector: vector,  // Skip re-embedding
   *   metadata: { optimized: true }
   * })
   * ```
   *
   * @example Multi-tenant usage
   * ```typescript
   * const id = await brain.add({
   *   data: "Customer feedback",
   *   type: NounType.Message,
   *   service: "customer-portal",  // Multi-tenancy
   *   metadata: { rating: 5, verified: true }
   * })
   * ```
   */
  /**
   * @description Mint a fresh, time-ordered entity id (UUID v7) WITHOUT a write.
   * Assign ids client-side to reference an entity before it exists — forward
   * references in `transact()`, deterministic `relate()`, no write→get→relate
   * round-trip. Time-ordered, so ids sort by creation time.
   * @returns A new UUID v7 string.
   * @example
   * const id = brain.newId()
   * await brain.add({ id, type: NounType.Document, data: 'draft' })
   */
  newId(): string {
    return uuidv7()
  }

  /**
   * @description Persist one single-operation write (`add`/`update`/`remove`/
   * `relate`/`updateRelation`/`unrelate` and the per-item calls of `addMany`/
   * `updateMany`/`relateMany`) as its OWN immutable generation — the Model-B
   * "every write versioned" contract. (`removeMany` is the one exception: it
   * commits each delete *chunk* as a single generation for bulk-delete efficiency,
   * not one generation per removed id.) Wraps the write's existing `executeTransaction` batch in
   * {@link GenerationStore.commitSingleOp}, which captures byte-identical
   * before-images of `touched`, runs the batch with the generation watermark
   * pinned to the reserved generation (so this write's graph-index ops stamp cor
   * with that one distinct per-op generation, via the unchanged
   * {@link graphWriteGeneration}), and buffers the history for async
   * group-commit. A crash before the flush loses only that buffered history, not
   * the acknowledged live write (drop-without-restore recovery).
   *
   * @param touched - The entity/relationship ids the write creates, updates, or
   *   deletes — the before-image + per-id-chain set.
   * @param run - The single-op's existing operation batch builder (the
   *   `tx => {…}` body previously passed straight to `executeTransaction`).
   * @param precommit - Optional CAS precondition, run under the commit mutex.
   * @param pendingEvents - Change-feed events to stamp and emit post-commit.
   * @param records - Optional v2 marker records (e.g. the deferred-embedding
   *   lifecycle markers) riding this write's commit fact — same generation,
   *   one atomic append. Refused on generation-less bootstrap writes.
   */
  /**
   * Storage-root-relative prefix of the RETIRED sidecar pending-embed marker
   * files (pre-log builds persisted one raw object per pending embed here).
   * The markers live IN the generation log now (`embed.pending` /
   * `embed.landed` records); this prefix survives ONLY for the one-time
   * migration bridge ({@link bridgeLegacyPendingEmbedSidecars}) — no other
   * code path writes, lists, or deletes it.
   */
  private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/'

  /**
   * @description Mark a deferred embed pending (MT5): the id joins the
   * in-memory fast-path set and the returned `embed.pending` record is
   * threaded onto the deferred write's OWN commit fact — same generation,
   * same atomic append, and (in at-ack log durability) the same covering
   * fsync as the write itself. The marker can never be orphaned from its
   * write nor the write from its marker: a failed commit appends no fact,
   * so no durable marker exists either (the in-memory entry is harmless
   * and reaped by the worker). Recovery folds the marker back out of the
   * log at open ({@link recoverPendingEmbedsFromLog}).
   */
  private enqueuePendingEmbed(id: string): FactMarkerRecord {
    this._pendingEmbedIds.add(id)
    return { type: 'embed.pending', id, enqueuedAt: Date.now() }
  }

  /**
   * @description Clear a pending embed from the in-memory set. The DURABLE
   * clear is the `embed.landed` record riding the landing commit's own fact
   * (or, for a row deleted before its embed landed, the row's tombstone
   * fact) — the recovery fold consumes those; nothing here touches storage.
   * One honest residue: a pending row whose entity still exists but carries
   * no data is reaped in memory only, so it re-folds at the next open and
   * is re-reaped there — a bounded no-op, never a lost vector.
   */
  private clearPendingEmbed(id: string): void {
    this._pendingEmbedIds.delete(id)
  }

  /**
   * @description Rebuild the pending-embed set by REPLAYING the generation
   * log's marker records (recovery = replay, not listing): `embed.pending`
   * arms an id, `embed.landed` disarms it, and a noun tombstone disarms it
   * too (a row deleted before its embed landed owes no vector). What
   * survives the fold is exactly the set of acknowledged deferred writes
   * whose vectors have not landed.
   *
   * BOUND (honest): no durable low-water mark exists for the earliest
   * unconsumed pending, so the fold scans the log's committed facts from
   * generation 1 — a sequential read of the log at open, O(log bytes).
   * It is SKIPPED WHOLESALE when the log has never had a v2 tail
   * ({@link FactLog.hasV2History} — v1 facts cannot carry marker records),
   * so pre-cutover brains pay nothing; on a mixed log the scan still reads
   * the v1 segments (a segment's format is only known from its bytes) but
   * they fold to nothing, so the DECODE cost is bounded by v2 history.
   * Storage without a fact log hosts no durable markers at all — the
   * pending set is session-local there, matching that storage's overall
   * durability posture.
   */
  private async recoverPendingEmbedsFromLog(): Promise<void> {
    const log = this.generationStore.getFactLog()
    if (!log || !log.hasV2History()) return
    const scan = log.scanFacts({ fromGeneration: 1 })
    for await (const batch of scan.batches()) {
      for (const fact of batch.facts) {
        for (const record of fact.records ?? []) {
          if (record.type === 'embed.pending') {
            this._pendingEmbedIds.add(record.id)
          } else if (record.type === 'embed.landed') {
            this._pendingEmbedIds.delete(record.id)
          }
        }
        for (const op of fact.ops) {
          if (op.kind === 'noun' && op.record === null) {
            this._pendingEmbedIds.delete(op.id)
          }
        }
      }
    }
  }

  /**
   * @description ONE-TIME LEGACY BRIDGE: a brain that deferred embeds under
   * a pre-log build persisted one sidecar marker file per pending embed
   * under {@link PENDING_EMBED_PREFIX}. At open, fold those ids into the
   * pending set AND migrate them: commit ONE fact carrying their
   * `embed.pending` records (the log is the markers' durable home now),
   * then delete the sidecar files — in that order, so a crash between the
   * two re-runs the bridge instead of losing a marker (a re-migrated
   * duplicate folds idempotently; at worst an already-landed embed re-runs
   * once — idempotent, never lost). Narrated loudly. Storage without a
   * fact log keeps its sidecars in place (there is no log to migrate into)
   * and folds them into memory only, exactly as loud.
   */
  private async bridgeLegacyPendingEmbedSidecars(): Promise<void> {
    const markerPaths = await this.storage.listRawObjects(Brainy.PENDING_EMBED_PREFIX)
    if (markerPaths.length === 0) return
    const ids: string[] = []
    for (const path of markerPaths) {
      const id = path.slice(path.lastIndexOf('/') + 1)
      if (id) ids.push(id)
    }
    if (ids.length === 0) return
    for (const id of ids) this._pendingEmbedIds.add(id)
    if (!this.generationStore.getFactLog()) {
      prodLog.warn(
        `[Brainy] ${ids.length} legacy pending-embed sidecar marker(s) found, but this ` +
          `storage hosts no fact log to migrate them into — folded into memory; the ` +
          `sidecar files remain the durable recovery source on this configuration`
      )
      return
    }
    const enqueuedAt = Date.now()
    const markers: FactMarkerRecord[] = ids.map((id) => ({
      type: 'embed.pending',
      id,
      enqueuedAt
    }))
    // One migration commit: a zero-op fact carrying every legacy marker
    // (empty-ops facts are legal; the records leg makes this one visible).
    await this.generationStore.commitSingleOp({
      touched: {},
      records: markers,
      execute: async () => {}
    })
    for (const id of ids) {
      await this.storage.deleteRawObject(`${Brainy.PENDING_EMBED_PREFIX}${id}`).catch(() => {})
    }
    prodLog.info(
      `[Brainy] migrated ${ids.length} legacy pending-embed sidecar marker(s) into the ` +
        `generation log and removed the sidecar files (one-time bridge)`
    )
  }

  /**
   * @description Start (or skip into) the ONE deferred-embedding worker.
   * Never awaited by write paths; failures are LOUD and markers survive for
   * the next kick (next deferred write, or the next open's recovery).
   */
  private kickEmbedWorker(): void {
    if (this._embedWorkerFlight || this._pendingEmbedIds.size === 0 || this.isReadOnly) return
    this._embedWorkerFlight = this.runEmbedWorker()
      .catch((err) => {
        prodLog.error(
          `[Brainy] deferred-embed worker failed: ${(err as Error).message} — ` +
            `markers retained; retries at the next deferred write or open`
        )
      })
      .finally(() => {
        this._embedWorkerFlight = null
        if (this._pendingEmbedIds.size > 0) {
          // New arrivals during the run: schedule (never recurse) the next pass.
          const t = setTimeout(() => this.kickEmbedWorker(), 0)
          ;(t as { unref?: () => void }).unref?.()
        }
      })
  }

  /**
   * @description Drain the pending-embed set: embed each row's CURRENT data
   * (a row updated again before its turn embeds the latest content — the
   * marker set is idempotent per id) and swap the vector in ATOMICALLY
   * (ReplaceInVectorIndex → the in-place update; the row is never absent
   * from search). Orphans (row deleted, or no data) reap their markers.
   */
  private async runEmbedWorker(): Promise<void> {
    const batch = Array.from(this._pendingEmbedIds)
    for (const id of batch) {
      try {
        const entity = await this.get(id, { includeVectors: true })
        if (!entity || entity.data === undefined || entity.data === null) {
          // Orphan reap: a deleted row's tombstone fact durably disarms the
          // marker at the next recovery fold; a data-less-but-present row
          // (edge case) re-folds and re-reaps — bounded, never a lost vector.
          this.clearPendingEmbed(id)
          continue
        }
        // Hang guard: a wedged embedder must not block every later pending
        // embed forever — time out LOUDLY, keep the marker, move on. (A
        // failure is retryable; an unbounded silent wait is the outlawed
        // shape.)
        const newVector = await Promise.race([
          this.embed(entity.data),
          new Promise<never>((_, reject) => {
            const t = setTimeout(
              () => reject(new Error('deferred embed timed out after 60s')),
              60_000
            )
            ;(t as { unref?: () => void }).unref?.()
          })
        ])
        if (!this.dimensions) {
          this.dimensions = newVector.length
        } else if (newVector.length !== this.dimensions) {
          throw new Error(
            `deferred embed produced ${newVector.length} dimensions, store expects ${this.dimensions}`
          )
        }
        const oldVector = (entity.vector as number[] | undefined) ?? []
        // The landing commit's fact carries the embed.landed record (vector
        // inline, per the v2 format) alongside the row's after-image — the
        // durable "this pending is consumed" that recovery's fold reads.
        await this.persistSingleOp(
          { nouns: [id] },
          async (tx) => {
            tx.addOperation(
              new SaveNounOperation(this.storage, {
                id,
                vector: newVector,
                connections: new Map(),
                level: 0
              })
            )
            tx.addOperation(
              new ReplaceInVectorIndexOperation(this.index, id, oldVector, newVector, this.indexWriteGeneration)
            )
          },
          undefined,
          undefined,
          [{ type: 'embed.landed', id, vector: newVector }],
          'system:embed-landing'
        )
        // Vectored-noun ledger: the landing commit above carries a vector
        // write with NO accompanying metadata operation, so the
        // saveNounMetadata(..., hasVector) seam never fires for it — the
        // narrow storage hook is the only seam left. `oldVector.length===0`
        // (already known for free from the pre-embed read above) proves this
        // is a GENUINE first landing, not a re-embed of an already-vectored
        // row (e.g. a deferred update() on a row that already had a real
        // vector) — the latter must never double-count.
        if (oldVector.length === 0) {
          await this.storage.noteVectorLanded?.(id)
        }
        this.clearPendingEmbed(id)
      } catch (err) {
        prodLog.warn(
          `[Brainy] deferred embed for ${id} failed: ${(err as Error).message} — marker retained for retry`
        )
      }
    }
  }

  /**
   * @description The deferred-embedding BARRIER: resolves when every pending
   * embed has landed (vector searchable) or been reaped. The eventual-
   * vector-index contract's awaitable edge — tests and "must be searchable
   * before I proceed" callers use this; nothing else ever needs to wait.
   */
  public async awaitPendingEmbeds(): Promise<void> {
    while (this._pendingEmbedIds.size > 0 || this._embedWorkerFlight) {
      this.kickEmbedWorker()
      await (this._embedWorkerFlight ?? Promise.resolve())
    }
  }

  /** The deferred-embedding backlog size (also on getIndexStatus().pendingEmbeds). */
  public pendingEmbedCount(): number {
    return this._pendingEmbedIds.size
  }

  /**
   * THE READ BARRIER: wait until a projection — or every projection — has
   * caught up to the CURRENT committed head, so a write-then-recall caller
   * has ONE honest await instead of a sleep-and-hope.
   *
   * Legs:
   * - `'semantic'` — waits for the deferred-embedding backlog to drain
   *   (delegates to {@link awaitPendingEmbeds}, which keeps working
   *   unchanged as this leg's engine). After it resolves, every previously
   *   acknowledged write is vector-searchable.
   * - `'metadata'` / `'graph'` / `'aggregation'` — resolve IMMEDIATELY by
   *   design today: these projections are updated inside the write path, so
   *   by the time a write's promise resolves they already reflect it. Their
   *   asynchrony arrives with the log-authority read path; the door's shape
   *   freezes now so callers written against it keep working unchanged when
   *   those legs become real waits.
   * - no argument — every projection at the head; today that reduces to the
   *   semantic drain (the only asynchronous projection in the current
   *   architecture).
   *
   * `opts.generation`: resolve as soon as the projection's watermark has
   * reached that committed generation. The pending-embed set carries no
   * generation stamps today, so the refinement is conservative — an empty
   * backlog resolves immediately (the watermark is at the head, hence ≥ any
   * committed generation); a non-empty backlog waits for the full drain, a
   * SUPERSET of the requested wait, never a partial one.
   *
   * `opts.timeoutMs`: on expiry the promise REJECTS with
   * {@link WaitForIndexedTimeoutError} — typed, carrying the leg and the
   * still-pending embed count, and naming the gauge to check
   * (`getIndexStatus().projections.semantic.pendingEmbeds`). Never a silent
   * partial wait: a timeout means the projection has NOT caught up.
   *
   * @example Write, then semantically recall — no polling, no sleeps
   * ```typescript
   * const id = await brain.add({
   *   data: 'quarterly revenue narrative',
   *   type: NounType.Document,
   *   deferEmbedding: true,
   *   metadata: { kind: 'report' }
   * })
   * await brain.waitForIndexed('semantic')   // the barrier: vector landed + indexed
   * const hits = await brain.find({ query: 'revenue report', searchMode: 'semantic' })
   * // `id` is eligible to appear in `hits` — the recall is honest, not lucky.
   * ```
   *
   * @param path - The projection to wait on; omit to wait on all of them.
   * @param opts - Optional `generation` watermark target and `timeoutMs` bound.
   * @throws {WaitForIndexedTimeoutError} When `timeoutMs` expires before the
   *   projection catches up.
   */
  public async waitForIndexed(
    path?: IndexedProjectionPath,
    opts?: WaitForIndexedOptions
  ): Promise<void> {
    await this.ensureInitialized()

    // Synchronous projections: updated inside the write path today, so an
    // acknowledged write is already reflected — resolve immediately BY
    // DESIGN (honest, not a stub). When the log-authority read path makes
    // these legs asynchronous, only this body changes; the door's shape is
    // frozen now.
    if (path === 'metadata' || path === 'graph' || path === 'aggregation') {
      return
    }

    // 'semantic' — or no-arg, which today reduces to it: the deferred-embed
    // backlog is the only asynchronous projection in the current
    // architecture.

    // Generation refinement (conservative — see JSDoc): an empty backlog
    // means the semantic watermark is at the head, hence ≥ any committed G.
    if (opts?.generation !== undefined && this._pendingEmbedIds.size === 0) {
      return
    }

    const timeoutMs = opts?.timeoutMs
    const drained = this.awaitPendingEmbeds()
    if (timeoutMs === undefined) {
      return drained
    }

    // Typed timeout: reject LOUDLY with the leg + the live backlog gauge.
    // (`drained` never rejects — the worker catches its own failures — so
    // abandoning it on timeout cannot leak an unhandled rejection; the
    // backlog keeps draining in the background.)
    let timer: ReturnType<typeof setTimeout> | undefined
    try {
      await Promise.race([
        drained,
        new Promise<never>((_, reject) => {
          timer = setTimeout(
            () =>
              reject(
                new WaitForIndexedTimeoutError(
                  path ?? 'all',
                  timeoutMs,
                  this._pendingEmbedIds.size
                )
              ),
            timeoutMs
          )
          ;(timer as { unref?: () => void }).unref?.()
        })
      ])
    } finally {
      if (timer !== undefined) clearTimeout(timer)
    }
  }

  /**
   * @description The write-side persistence trigger (policy `'auto'`): count
   * the committed write, kick a single-flight BACKGROUND flush when the
   * write-count or interval threshold is crossed, and (re)arm the idle
   * timer. Never awaited by the write path — the ack is already durable at
   * the canonical layer; this schedules DERIVED-state persistence on the
   * engine's own cadence (callers never call flush() in hot paths).
   */
  private noteWriteForPersistence(): void {
    // THE DIRTY WITNESS. Set on every committed write — both commit paths
    // (single-op and transaction) end here, and the deferred-embed worker
    // lands its vectors through the single-op path — BEFORE the policy check,
    // so a `'manual'` consumer's explicit flush() is never skipped either.
    // Cleared by a flush that actually runs; see flush().
    this._dirtySinceLastFlush = true
    const cfg = this.config.persistence
    if (this.isReadOnly || cfg?.policy === 'manual') return
    this._persistDirtyWrites++
    const every = cfg?.flushEveryWrites ?? 512
    const intervalMs = cfg?.flushIntervalMs ?? 30_000
    const idleMs = cfg?.flushOnIdleMs ?? 2_000

    if (
      this._persistDirtyWrites >= every ||
      Date.now() - this._persistLastFlushAt >= intervalMs
    ) {
      this.kickBackgroundFlush('threshold')
    }

    if (this._persistIdleTimer) clearTimeout(this._persistIdleTimer)
    this.armIdleFlushTimer(idleMs, intervalMs)
  }

  /**
   * @description Arm the idle-flush timer — DEBOUNCED UNDER LOAD. The idle
   * trigger exists to make a QUIET system durable fast; it must never add
   * flush pressure to a BUSY one. When individual writes are slower than
   * the idle window (a contended disk), every inter-write gap looks like
   * "idle" and would fire a full flush per write — a measured 15-flush
   * amplifier during 100 contended adds on a production-shaped box. The
   * law: an idle fire landing within `intervalMs` of the last flush DEFERS
   * (re-arms for the remaining interval) rather than flushing — deferred,
   * never dropped, so a lone write on a then-quiet system still persists at
   * the interval boundary without any further write arriving; a genuinely
   * quiet system (last flush long past) flushes on idle exactly as before.
   */
  private armIdleFlushTimer(idleMs: number, intervalMs: number, delayMs = idleMs): void {
    // The idle-fire spacing floor: 10× the CONFIGURED idle window, capped by
    // the interval — always derived from idleMs, never from a deferred
    // re-arm delay (recomputing from the delay compounds into runaway
    // deferral). Scales with intent — a caller configuring a tiny idle
    // window gets fast idle-driven durability (small floor); default config
    // (2s idle / 30s interval) gets a 20s floor, capping the contended-disk
    // shape at ~1 idle flush per 20s instead of one per inter-write gap.
    const floorMs = Math.min(intervalMs, idleMs * 10)
    const timer = setTimeout(() => {
      this._persistIdleTimer = null
      if (this._persistDirtyWrites === 0) return
      const sinceFlush = Date.now() - this._persistLastFlushAt
      if (sinceFlush >= floorMs) {
        this.kickBackgroundFlush('idle')
      } else {
        // Deferred, never dropped: land exactly at the floor boundary.
        this.armIdleFlushTimer(idleMs, intervalMs, Math.max(idleMs, floorMs - sinceFlush))
      }
    }, delayMs)
    // Never hold the process open for a cadence timer.
    ;(timer as { unref?: () => void }).unref?.()
    this._persistIdleTimer = timer
  }

  /**
   * @description Start (or join) the ONE background flush. The dirty counter
   * resets at kick time so writes landing during the flush re-accumulate
   * toward the next trigger. A failure is LOUD and leaves the writes counted
   * again — silence is not an option, and neither is a retry storm (the next
   * trigger re-attempts).
   */
  private kickBackgroundFlush(reason: 'threshold' | 'idle'): void {
    if (this._persistBackgroundFlight) return
    const counted = this._persistDirtyWrites
    this._persistDirtyWrites = 0
    this._persistLastFlushAt = Date.now()
    this._persistBackgroundFlight = this.flush()
      .catch((err) => {
        this._persistDirtyWrites += counted // re-arm the trigger honestly
        prodLog.error(
          `[Brainy] background flush (${reason}) FAILED: ${(err as Error).message} — ` +
            `derived-state persistence retries at the next trigger; canonical data is unaffected`
        )
      })
      .finally(() => {
        this._persistBackgroundFlight = null
      })
  }

  private async persistSingleOp(
    touched: { nouns?: string[]; verbs?: string[] },
    run: TransactionFunction<void>,
    precommit?: (before: CommitBeforeImages) => void,
    pendingEvents?: PendingChangeEvent[],
    records?: FactMarkerRecord[],
    origin?: string
  ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> {
    // Change-feed capture: when this write will emit, hold a reference to the
    // commit's before-images so `remove` events can carry the record's last
    // committed state (free — the commit reads them anyway for Model B).
    let capturedBefore: CommitBeforeImages | undefined
    const captureAndCheck =
      pendingEvents && pendingEvents.length > 0
        ? (before: CommitBeforeImages): void => {
            capturedBefore = before
            precommit?.(before)
          }
        : precommit

    if (!this._generationStampingActive) {
      // Marker records ride a commit FACT — a generation-less bootstrap
      // write has none to ride. No bootstrap path defers embeds today;
      // refuse loudly rather than silently dropping a durable marker.
      if (records && records.length > 0) {
        throw new Error(
          'persistSingleOp: marker records require a generation-stamped commit — ' +
            'a bootstrap (generation-0) write cannot carry them'
        )
      }
      // Init-time / infrastructure baseline write (e.g. the VFS root): apply
      // WITHOUT creating a generation. Generation 0 is the freshly-materialized
      // brain (bootstrap included); the first USER write is generation 1.
      // Bootstrap writes are single-threaded, so a supplied precondition is
      // honored against directly-read before-images (no commit mutex exists
      // on this path — nothing to race).
      if (captureAndCheck) {
        const nouns = new Map<string, { kind: 'noun'; metadata: unknown; vector: unknown }>()
        for (const id of touched.nouns ?? []) {
          const prev = await this.storage.readNounRaw(id)
          nouns.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector })
        }
        const verbs = new Map<string, { kind: 'verb'; metadata: unknown; vector: unknown }>()
        for (const id of touched.verbs ?? []) {
          const prev = await this.storage.readVerbRaw(id)
          verbs.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector })
        }
        captureAndCheck({ nouns, verbs } as CommitBeforeImages)
      }
      await this.generationStore.runWithoutGeneration(() =>
        this.transactionManager.executeTransaction(run, {
          timeout: transactTimeoutBudget(
            (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0),
            undefined,
            this.config.transactionBudgetFloorMs
          )
        })
      )
      const timestamp = Date.now()
      // Bootstrap writes are not generation-stamped; emit without one.
      this.emitCommitted(pendingEvents, capturedBefore, undefined, timestamp)
      return { timestamp }
    }
    let receipt
    try {
      receipt = await this.generationStore.commitSingleOp({
        touched,
        precommit: captureAndCheck,
        ...(records && records.length > 0 ? { records } : {}),
        ...(origin ? { origin } : {}),
        execute: () =>
          this.transactionManager.executeTransaction(run, {
            timeout: transactTimeoutBudget(
              (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0),
              undefined,
              this.config.transactionBudgetFloorMs
            )
          })
      })
    } catch (err) {
      // A failed rollback that left the store inconsistent (a remove/update
      // whose restore-undo failed) quarantines writes until repairIndex().
      if (err instanceof StoreInconsistentError) this.storeInconsistency = err
      throw err
    }
    // POST-COMMIT ONLY: an aborted commit (CAS conflict, failed apply) throws
    // above and never reaches this line — the feed cannot announce a write
    // that did not become durable. (A degraded adopt-forward write DID commit —
    // it carries a generation and emits normally.)
    this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp)
    // An adopt-forward failed-rollback recovery committed the record but may have
    // left its derived index incomplete (generationStore already warned once).
    // Record the ids so reads and checkHealth() surface the incompleteness rather
    // than silently returning partial data; repairIndex() clears them.
    if (receipt.degraded && receipt.degraded.length > 0) {
      for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId)
      this._degradedReadWarned = false // re-arm the read-path warning
      if (!this.config.silent) {
        prodLog.warn(
          `[Brainy] A single-op write committed in a DEGRADED state — the derived ` +
          `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` +
          `Reads may return partial results until repairIndex() reconciles them.`
        )
      }
    }
    this.noteWriteForPersistence()
    return receipt
  }

  /**
   * @description Stamp pending change events with their commit receipt,
   * enrich entity `remove` events with the record's last committed state
   * (from the commit's before-images), and hand them to the change feed for
   * post-mutex dispatch. No-op when the write produced no events.
   * @param pending - Events the mutation method constructed pre-commit
   *   (only when a listener is subscribed — see {@link ChangeFeed.hasListeners}).
   * @param before - The commit's before-images (delete-payload source).
   * @param generation - The committed generation (absent for bootstrap writes).
   * @param timestamp - The commit timestamp.
   */
  private emitCommitted(
    pending: PendingChangeEvent[] | undefined,
    before: CommitBeforeImages | undefined,
    generation: number | undefined,
    timestamp: number
  ): void {
    if (!pending || pending.length === 0) return
    const events: BrainyChangeEvent[] = pending.map((p) => {
      let entity = p.entity
      if (!entity && p.kind === 'entity' && p.op === 'remove' && p.id && before) {
        const record = before.nouns.get(p.id)?.metadata as
          | Record<string, unknown>
          | null
          | undefined
        if (record) {
          entity = this.entityViewFromRawRecord(p.id, record)
        }
      }
      return {
        ...p,
        ...(entity && { entity }),
        ...(generation !== undefined && { generation }),
        timestamp
      }
    })
    this._changeFeed.emit(events)
  }

  /**
   * @description Build the change-event entity view from a stored flat
   * metadata record (the shape before-images and pre-delete reads carry),
   * using THE canonical reserved/custom split so the view can never drift
   * from the live read paths.
   * @param id - The entity's canonical UUID.
   * @param record - The stored flat metadata record.
   * @returns The `ChangeEventEntity` view (type, subtype, service, custom metadata).
   */
  private entityViewFromRawRecord(
    id: string,
    record: Record<string, unknown>
  ): NonNullable<BrainyChangeEvent['entity']> {
    const { reserved, custom } = splitNounMetadataRecord(record)
    return {
      id,
      type: String(reserved.noun ?? 'unknown'),
      ...(reserved.subtype !== undefined && { subtype: String(reserved.subtype) }),
      metadata: custom,
      ...(reserved.service !== undefined && { service: String(reserved.service) })
    }
  }

  /**
   * @description Build the AGGREGATION view of an entity from a stored flat
   * metadata record — EVERY reserved field mapped to its top-level entity
   * name (stored `noun` → `type`), custom metadata in `metadata`. This must
   * mirror the add-path `entityForIndexing` shape exactly: the aggregation
   * engine resolves groupBy/where fields via `resolveEntityField`
   * (top-level standard fields + custom metadata), so a view that drops a
   * reserved field makes every aggregate grouped by that field decrement a
   * group that does not exist — counts then drift upward forever after
   * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this.
   * @param record - The stored flat metadata record (before-image or pre-delete read).
   * @returns The full-fidelity entity view for aggregation hooks.
   */
  private entityForAggFromRawRecord(record: Record<string, unknown>): Record<string, unknown> {
    const { reserved, custom } = splitNounMetadataRecord(record)
    const { noun, ...rest } = reserved
    return {
      type: noun,
      ...rest,
      metadata: custom
    }
  }

  /**
   * @description Add an entity (noun) to the brain. Embeds `data` into a vector and
   * indexes the entity across all three intelligences — vector similarity, graph
   * adjacency, and metadata — then returns its id. When no `id` is supplied a fresh
   * time-ordered **UUID v7** is generated; a supplied natural-key string is
   * normalized to a stable **UUID v5**. Under Model-B every add is its own immutable
   * generation, so it is time-travelable via {@link asOf}.
   * @param params - The entity to add. `data` (embedded for similarity) and `type`
   *   (a {@link NounType}) are the essentials; `subtype`, `metadata`, an explicit
   *   `id`, `visibility`, `confidence`, and `weight` are optional.
   * @returns The entity's id — the supplied/normalized id, or a freshly generated UUID v7.
   * @throws If the brain is read-only (`mode: 'reader'`), or brain-wide `requireSubtype`
   *   is on and the entity's type has no subtype.
   * @example
   * const id = await brain.add({
   *   data: 'Ada Lovelace',
   *   type: NounType.Person,
   *   metadata: { role: 'mathematician' }
   * })
   */
  async add(params: AddParams<T>): Promise<string> {
    this.assertWritable('add')
    await this.ensureInitialized()

    // Zero-config validation (static import for performance)
    validateAddParams(params)


    // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a
    // tracked field declared at top level (e.g. 'subtype') and one declared in
    // metadata (e.g. 'status') both validate.
    this.enforceTrackedFieldValues(params.metadata as Record<string, unknown> | undefined, 'metadata')
    this.enforceTrackedFieldValues(
      { subtype: params.subtype } as Record<string, unknown>,
      'top-level'
    )

    // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered
    // via brain.requireSubtype() compose with the brain-wide strict-mode flag.
    // Metadata is passed so infrastructure writes (VFS) can bypass the
    // missing-subtype check via the `isVFSEntity` marker.
    this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata)

    // Id normalization (8.0): a natural string key (e.g. 'user-123') is coerced
    // to a STABLE UUID (v5) so the engine only ever sees a UUID, while the
    // caller's original string is preserved under ORIGINAL_ID_KEY for reads.
    // A real UUID passes through; no id mints a fresh v7. Done BEFORE the
    // ifAbsent check so a natural-key upsert is idempotent against the same key.
    const { id, originalId } = coerceNewEntityId(params.id)

    // ifAbsent and upsert are opposite resolutions of the same id collision —
    // ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both
    // would be ambiguous, so it is a hard error.
    if (params.ifAbsent && params.upsert) {
      throw new Error(
        'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
      )
    }

    // ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id
    // is supplied; a freshly generated UUID can never collide. Returns the existing
    // (canonical) id without writing if the entity is already present (no throw,
    // no overwrite). This check is a FAST-FAIL (skips the embedding cost); the
    // authoritative absence check runs in the insert's commit precondition
    // below, atomic with the apply, so a concurrent same-id create cannot make
    // both callers write.
    if (params.id && params.ifAbsent) {
      const existing = await this.storage.getNounMetadata(id)
      if (existing) return id
    }

    // upsert — by-ID create-or-update. Only meaningful when a custom id is supplied
    // (a freshly generated UUID can never collide). When the id already exists,
    // delegate to update() so the supplied fields MERGE into the existing entity
    // (metadata merge, re-embed on changed data, _rev bump, createdAt preserved)
    // instead of the default destructive overwrite. When the id is absent, fall
    // through to the guarded insert below (whose commit precondition converts a
    // concurrent same-id create into this same merge — never an overwrite).
    if (params.id && params.upsert) {
      const existing = await this.storage.getNounMetadata(id)
      if (existing) {
        await this.update(this.upsertMergeParams(id, params))
        return id
      }
    }

    // Get or compute vector
    // MT5 deferred embedding: ack at durability with a stub vector and a
    // pending marker riding the insert's OWN commit fact (same generation,
    // one atomic append — a marker-less committed row, the silently-missing-
    // vector shape, is structurally impossible). The background worker
    // embeds + inserts.
    const deferringEmbed = params.deferEmbedding === true && !params.vector
    let vector = deferringEmbed
      ? []
      : params.vector || (await this.embed(params.data))

    // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a
    // vector — it never crosses an engine boundary (the engine pair's seam
    // law). This engine's own cosine distance treats an all-zero vector
    // safely (a zero-norm operand always scores MAXIMUM distance — see
    // isZeroNormVector's JSDoc), but a downstream engine serving squared-
    // euclidean distance cannot tell it apart from a legitimate origin
    // point — a false attractor that silently darkened 150+ rows in a
    // production deployment. The index belt (AddToVectorIndexOperation)
    // already refuses to INDEX a zero-norm vector, but until now the
    // CANONICAL write still persisted it and the vectored-noun ledger
    // counted it — so a near-empty store whose only vectored row was
    // zero-norm read "canonical vectored > 0, index size 0" and threw a
    // not-ready error at open. Normalize HERE, before the dimension pin,
    // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`),
    // and the index ops below ever see it, so it persists as the sanctioned
    // "unvectored" `[]` shape instead — the canonical write still succeeds.
    if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) {
      prodLog.warn(
        `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` +
          `a zero-norm vector is not a vector; persisted unvectored ([]) instead.`
      )
      vector = []
    }

    // Ensure dimensions are set (a deferred-embed stub carries no dimension
    // information — the worker's real vector goes through the same guard).
    // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose
    // vector is the "unvectored" empty-array shape carries no dimension
    // information, deferred or not — an explicit `vector: []` (e.g. the VFS
    // root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must
    // never pin `this.dimensions` to 0, which would poison every subsequent
    // real embed's dimension check for the life of the store.
    if (!deferringEmbed && vector.length > 0) {
      if (!this.dimensions) {
        this.dimensions = vector.length
      } else if (vector.length !== this.dimensions) {
        throw new Error(
          `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}`
        )
      }
    }

    // Prepare metadata for storage: a v2 nested-bag record — engine fields
      // top-level, the user's bag nested VERBATIM (any name, including engine
      // spellings like `confidence` or `type`, is the user's and survives
      // faithfully; the field-addressing law).
      const storageMetadata = buildNounMetadataRecord(
        {
          data: params.data,
          noun: params.type,
          ...(params.subtype !== undefined && { subtype: params.subtype }),
          // visibility: stored only when not 'public' (absent === public, keeps records lean)
          ...(params.visibility !== undefined &&
            params.visibility !== 'public' && { visibility: params.visibility }),
          service: params.service,
          createdAt: Date.now(),
          updatedAt: Date.now(),
          _rev: 1,
          ...(params.confidence !== undefined && { confidence: params.confidence }),
          ...(params.weight !== undefined && { weight: params.weight }),
          ...(params.createdBy && { createdBy: params.createdBy })
        },
        {
          ...params.metadata,
          // Preserve the caller's original (non-UUID) id when normalized, so reads
          // can surface it. A real UUID passes through with no _originalId.
          ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId })
        }
      )

      // Build entity structure for indexing (NEW - with top-level fields)
      // Optional fields must use conditional spreading to match storageMetadata exactly.
      // If undefined values are included as explicit keys, extractIndexableFields indexes
      // them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata
      // omits those keys entirely via conditional spreading, so the fields don't match).
      // No `level` here: engine plumbing never enters the indexing view — a
      // hardcoded level:0 landed in the SAME flattened index column as user
      // metadata named `level`, poisoning it multi-valued ([0, real]).
      const entityForIndexing = {
        id,
        vector,
        connections: new Map(),
        type: params.type,
        ...(params.subtype !== undefined && { subtype: params.subtype }),
        ...(params.visibility !== undefined &&
          params.visibility !== 'public' && { visibility: params.visibility }),
        ...(params.confidence !== undefined && { confidence: params.confidence }),
        ...(params.weight !== undefined && { weight: params.weight }),
        createdAt: Date.now(),
        updatedAt: Date.now(),
        service: params.service,
        data: params.data,
        ...(params.createdBy && { createdBy: params.createdBy }),
        // Only custom fields in metadata (plus the preserved original id, which
        // is a custom field — surfaced on read under ORIGINAL_ID_KEY).
        metadata: {
          ...(params.metadata || {}),
          ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId })
        }
      }

      // Execute atomically with transaction system, generation-stamped as one
      // immutable Model-B generation (before-image = the absent/create sentinel).
      // All operations succeed or all rollback - prevents partial failures
      // ifAbsent/upsert insert leg: must-be-absent commit precondition, run
      // under the commit mutex against the authoritative before-image — the
      // planning-time absence checks above are fast-fails that can interleave
      // with a concurrent same-id create. Exactly one concurrent create wins;
      // every other caller takes its documented resolution instead of
      // silently overwriting the winner.
      const requireAbsent = Boolean(params.id && (params.ifAbsent || params.upsert))
      const insertPrecommit = requireAbsent
        ? (before: CommitBeforeImages): void => {
            if (before.nouns.get(id)?.metadata) {
              throw new InsertPreconditionExistsSignal(id)
            }
          }
        : undefined

      // MT5: the pending marker RIDES the insert's own commit fact (same
      // generation, one atomic append) — threaded to persistSingleOp below.
      // A failed commit appends nothing, so no orphaned durable marker can
      // exist; the in-memory entry is harmless and reaped by the worker.
      const embedMarkers: FactMarkerRecord[] | undefined = deferringEmbed
        ? [this.enqueuePendingEmbed(id)]
        : undefined

      const runInsert: TransactionFunction<void> = async (tx) => {
        // Operation 1: Save metadata FIRST (TypeAwareStorage caching)
        // isNew=true: skip pre-read for rollback (entity doesn't exist yet)
        // hasVector: the vectored-noun ledger counts this insert iff its
        // vector is real/non-empty (never true for a deferred embed, whose
        // stub `vector` is `[]` — it counts later, at landing).
        tx.addOperation(
          new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0)
        )

        // Operation 2: Save vector data
        // isNew=true: skip pre-read for rollback (entity doesn't exist yet)
        tx.addOperation(
          new SaveNounOperation(this.storage, {
            id,
            vector,
            connections: new Map(),
            level: 0
          }, true)
        )

        // Operation 3: Add to HNSW index (after entity saved). Gated on
        // `vector.length > 0`, not `!deferringEmbed`: a deferred embed has
        // nothing to index yet (the worker's atomic update inserts the real
        // vector later), and an explicit `vector: []` insert (the VFS root's
        // zero-norm fix — permanently unvectored plumbing, never embedded)
        // is exactly the same "nothing to index yet" shape. The zero-norm
        // BELT (a real all-zero vector, non-empty) is enforced inside
        // AddToVectorIndexOperation itself — see its JSDoc.
        if (vector.length > 0) {
          tx.addOperation(
            new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)
          )
        }

        // Operation 4: Add to metadata index
        tx.addOperation(
          new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration)
        )
      }

      // Bounded retry closes the remaining interleavings without ever holding
      // a lock across the loop: a lost insert race resolves to skip (ifAbsent)
      // or merge (upsert); a merge that then hits a concurrent delete retries
      // the insert. Each persistSingleOp call constructs fresh operations, so
      // re-running the callback is safe.
      // Change feed: built only when someone is listening (zero-cost gate).
      const addEvents: PendingChangeEvent[] | undefined = this._changeFeed.hasListeners
        ? [
            {
              kind: 'entity',
              op: 'add',
              id,
              entity: {
                id,
                type: String(params.type),
                ...(params.subtype !== undefined && { subtype: String(params.subtype) }),
                metadata: (params.metadata as Record<string, unknown>) ?? {},
                ...(params.service !== undefined && { service: String(params.service) })
              }
            }
          ]
        : undefined

      const MAX_UPSERT_ATTEMPTS = 10
      for (let attempt = 0; ; attempt++) {
        try {
          await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents, embedMarkers)
          break
        } catch (err) {
          if (!(err instanceof InsertPreconditionExistsSignal)) {
            throw err
          }
          // A concurrent writer created the entity between our absence check
          // and the commit.
          if (params.ifAbsent) {
            // Documented contract: return the existing id without writing.
            return id
          }
          // upsert: merge into the now-existing entity. A concurrent delete
          // between this conflict and the merge throws EntityNotFoundError —
          // loop back and retry the insert.
          try {
            await this.update(this.upsertMergeParams(id, params))
            return id
          } catch (mergeErr) {
            if (
              !(mergeErr instanceof EntityNotFoundError) ||
              attempt >= MAX_UPSERT_ATTEMPTS
            ) {
              throw mergeErr
            }
          }
        }
      }

      // Aggregation hook (outside transaction — derived data, can be reconstructed)
      if (this._aggregationIndex) {
        this._aggregationIndex.onEntityAdded(id, entityForIndexing)
      }

      if (deferringEmbed) this.kickEmbedWorker()
      return id
  }

  /**
   * @description The `update()` param mapping `add({ upsert })` delegates to
   * when the target id already exists: every supplied field merges into the
   * existing entity (metadata merge, re-embed on changed data, `_rev` bump,
   * `createdAt` preserved); absent fields are left untouched. Shared by the
   * planning-time upsert branch and the commit-conflict retry so the two
   * paths can never drift.
   * @param id - The canonical entity id the upsert resolved to.
   * @param params - The original `add()` params carrying the fields to merge.
   * @returns The `UpdateParams` for the merging `update()` call.
   */
  private upsertMergeParams(id: string, params: AddParams<T>): UpdateParams<T> {
    return {
      id,
      ...(params.data !== undefined && { data: params.data }),
      ...(params.type !== undefined && { type: params.type }),
      ...(params.subtype !== undefined && { subtype: params.subtype }),
      ...(params.visibility !== undefined && { visibility: params.visibility }),
      ...(params.metadata !== undefined && { metadata: params.metadata }),
      ...(params.vector !== undefined && { vector: params.vector }),
      ...(params.confidence !== undefined && { confidence: params.confidence }),
      ...(params.weight !== undefined && { weight: params.weight })
    } as UpdateParams<T>
  }

  /**
   * Get an entity by ID
   *
   * @param id - The unique identifier of the entity to retrieve
   * @returns Promise that resolves to the entity if found, null if not found
   *
   * **Entity includes:**
   * - `confidence` - Type classification confidence (0-1) if set
   * - `weight` - Entity importance/salience (0-1) if set
   * - All standard fields: id, type, data, metadata, vector, timestamps
   *
   * @example
   * // Basic entity retrieval
   * const entity = await brainy.get('user-123')
   * if (entity) {
   *   console.log('Found entity:', entity.data)
   *   console.log('Created at:', new Date(entity.createdAt))
   * } else {
   *   console.log('Entity not found')
   * }
   *
   * @example
   * // Accessing confidence and weight
   * const entity = await brainy.get('concept-456')
   * if (entity) {
   *   console.log(`Type: ${entity.type}`)
   *   console.log(`Confidence: ${entity.confidence ?? 'N/A'}`)
   *   console.log(`Weight: ${entity.weight ?? 'N/A'}`)
   * }
   *
   * @example
   * // Working with typed entities
   * interface User {
   *   name: string
   *   email: string
   * }
   *
   * const brainy = new Brainy<User>({ storage: 'filesystem' })
   * const user = await brainy.get('user-456')
   * if (user) {
   *   // TypeScript knows user.metadata is of type User
   *   console.log(`Hello ${user.metadata.name}`)
   * }
   *
   * @example
   * // Safe retrieval with error handling
   * try {
   *   const entity = await brainy.get('document-789')
   *   if (!entity) {
   *     throw new Error('Document not found')
   *   }
   *
   *   // Process the entity
   *   return {
   *     id: entity.id,
   *     content: entity.data,
   *     type: entity.type,
   *     metadata: entity.metadata
   *   }
   * } catch (error) {
   *   console.error('Failed to retrieve entity:', error)
   *   return null
   * }
   *
   * @example
   * // Batch retrieval pattern
   * const ids = ['doc-1', 'doc-2', 'doc-3']
   * const entities = await Promise.all(
   *   ids.map(id => brainy.get(id))
   * )
   * const foundEntities = entities.filter(entity => entity !== null)
   * console.log(`Found ${foundEntities.length} out of ${ids.length} entities`)
   *
   * @example
   * // Using with async iteration
   * const entityIds = ['user-1', 'user-2', 'user-3']
   *
   * for (const id of entityIds) {
   *   const entity = await brainy.get(id)
   *   if (entity) {
   *     console.log(`Processing ${entity.type}: ${id}`)
   *     // Process entity...
   *   }
   * }
   */
  /**
   * Get an entity by ID
   *
   * **Performance**: Optimized for metadata-only reads by default
   * - **Default (metadata-only)**: 10ms, 300 bytes - 76-81% faster
   * - **Full entity (includeVectors: true)**: 43ms, 6KB - when vectors needed
   *
   * **When to use metadata-only (default)**:
   * - VFS operations (readFile, stat, readdir) - 100% of cases
   * - Existence checks: `if (await brain.get(id))`
   * - Metadata inspection: `entity.metadata`, `entity.data`, `entity.type`
   * - Relationship traversal: `brain.related({ from: id })`
   *
   * **When to include vectors**:
   * - Computing similarity on this specific entity: `brain.similar({ to: entity.vector })`
   * - Manual vector operations: `cosineSimilarity(entity.vector, otherVector)`
   *
   * @param id - Entity ID to retrieve
   * @param options - Retrieval options (includeVectors defaults to false)
   * @returns Entity or null if not found
   *
   * @example
   * ```typescript
   * // ✅ FAST: Metadata-only (default) - 10ms, 300 bytes
   * const entity = await brain.get(id)
   * console.log(entity.data, entity.metadata)  // ✅ Available
   * console.log(entity.vector.length)  // 0 (stub vector)
   *
   * // ✅ FULL: Include vectors when needed - 43ms, 6KB
   * const fullEntity = await brain.get(id, { includeVectors: true })
   * const similarity = cosineSimilarity(fullEntity.vector, otherVector)
   *
   * // ✅ Existence check (metadata-only is perfect)
   * if (await brain.get(id)) {
   *   console.log('Entity exists')
   * }
   *
   * // ✅ VFS automatically benefits (no code changes needed)
   * await vfs.readFile('/file.txt')  // 53ms → 10ms (81% faster)
   * ```
   *
   * @performance
   * - Metadata-only: 76-81% faster, 95% less bandwidth, 87% less memory
   * - Full entity: Same (no regression)
   * - VFS operations: 81% faster with zero code changes
   *
   */
  async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
    // Canonical read: a get resolves an entity by id straight from storage and
    // consults no derived index — it must not wait on any family's migration.
    await this.ensureInitialized({ needs: [] })
    this.warnIfReadsDegraded('get')

    // Id normalization (8.0): a caller may read by their natural key — resolve
    // it to the same canonical UUID add() stored. A real UUID passes through.
    // Only a non-empty string is normalized; empty/null/undefined fall through
    // to the storage layer's structural-validation throw (preserved contract).
    if (typeof id === 'string' && id !== '') {
      id = resolveEntityId(id)
    }

    // Route to metadata-only or full entity based on options
    const includeVectors = options?.includeVectors ?? false  // Default: metadata-only (fast)

    if (includeVectors) {
      // FULL PATH: Load vector + metadata (6KB, 43ms)
      // Used when: Computing similarity on this entity, manual vector operations
      const noun = await this.storage.getNoun(id)
      if (!noun) {
        return null
      }
      return this.convertNounToEntity(noun)
    } else {
      // FAST PATH: Metadata-only (300 bytes, 10ms) - DEFAULT
      // Used when: VFS operations, existence checks, metadata inspection (94% of calls)
      const metadata = await this.storage.getNounMetadata(id)
      if (!metadata) {
        return null
      }
      return this.convertMetadataToEntity(id, metadata)
    }
  }

  /**
   * Batch get multiple entities by IDs (Cloud Storage Optimization)
   *
   * **Performance**: Eliminates N+1 query pattern
   * - Current: N × get() = N × 300ms cloud latency = 3-6 seconds for 10-20 entities
   * - Batched: 1 × batchGet() = 1 × 300ms cloud latency = 0.3 seconds ✨
   *
   * **Use cases:**
   * - VFS tree traversal (get all children at once)
   * - Relationship traversal (get all targets at once)
   * - Import operations (batch existence checks)
   * - Admin tools (fetch multiple entities for listing)
   *
   * @param ids Array of entity IDs to fetch
   * @param options Get options (includeVectors defaults to false for speed)
   * @returns Map of id → entity (only successfully fetched entities included)
   *
   * @example
   * ```typescript
   * // VFS getChildren optimization
   * const childIds = relations.map(r => r.to)
   * const childrenMap = await brain.batchGet(childIds)
   * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean)
   * ```
   */
  async batchGet(ids: string[], options?: GetOptions): Promise<Map<string, Entity<T>>> {
    // Canonical read (see get): resolves by id from storage, no derived index.
    await this.ensureInitialized({ needs: [] })

    // Id normalization (8.0): resolve each id to its canonical UUID so callers
    // can batch-read by natural key. resolveEntityId is idempotent on real
    // UUIDs, so internal callers passing canonical ids are unaffected. The
    // returned map is keyed by the canonical (stored) id.
    ids = ids.map((id) => resolveEntityId(id))

    const results = new Map<string, Entity<T>>()
    if (ids.length === 0) return results

    const includeVectors = options?.includeVectors ?? false

    if (includeVectors) {
      // FULL PATH optimized with batch vector loading (10x faster on GCS)
      // GCS: 10 entities with vectors = 1×50ms vs 10×50ms = 500ms (10x faster)
      const nounsMap = await this.storage.getNounBatch(ids)

      for (const [id, noun] of nounsMap.entries()) {
        const entity = await this.convertNounToEntity(noun)
        results.set(id, entity)
      }
    } else{
      // FAST PATH: Metadata-only batch (default) - OPTIMIZED
      const metadataMap = await this.storage.getNounMetadataBatch(ids)

      for (const [id, metadata] of metadataMap.entries()) {
        const entity = await this.convertMetadataToEntity(id, metadata)
        results.set(id, entity)
      }
    }

    return results
  }

  /**
   * Create a flattened Result object from entity
   * Flattens commonly-used entity fields to top level for convenience
   */
  private createResult(id: string, score: number, entity: Entity<T>, explanation?: ScoreExplanation): Result<T> {
    return {
      id,
      score,
      // Flatten common entity fields to top level
      type: entity.type,
      subtype: entity.subtype,
      visibility: entity.visibility,
      metadata: entity.metadata,
      data: entity.data,
      confidence: entity.confidence,
      weight: entity.weight,
      _rev: entity._rev,
      // Preserve full entity for backward compatibility
      entity,
      // Optional score explanation
      ...(explanation && { explanation })
    }
  }

  /**
   * Convert a noun from storage to an entity (SIMPLIFIED!)
   *
   * Dramatically simplified - standard fields moved to top-level
   * - Extracts standard fields from metadata (storage format)
   * - Returns entity with standard fields at top-level (in-memory format)
   * - metadata contains ONLY custom user fields
   */
  private async convertNounToEntity(noun: any): Promise<Entity<T>> {
    // Storage adapters ALREADY extract standard fields to top-level!
    // Just read from top-level fields of HNSWNounWithMetadata

    // Clean structure with standard fields at top-level
    const entity: Entity<T> = {
      id: noun.id,
      vector: noun.vector,
      type: noun.type || NounType.Thing,
      subtype: noun.subtype,
      visibility: noun.visibility,

      // Standard fields at top-level
      confidence: noun.confidence,
      weight: noun.weight,
      createdAt: noun.createdAt || Date.now(),
      updatedAt: noun.updatedAt || Date.now(),
      service: noun.service,
      data: noun.data,
      createdBy: noun.createdBy,
      // 7.31.0 — surface revision counter (defaults to 1 for pre-7.31.0 entities)
      _rev: typeof noun._rev === 'number' ? noun._rev : 1,

      // ONLY custom user fields in metadata (already separated by storage adapter)
      metadata: noun.metadata as T
    }

    return entity
  }

  /**
   * Convert metadata-only to entity (FAST PATH!)
   *
   * Used when vectors are NOT needed (94% of brain.get() calls):
   * - VFS operations (readFile, stat, readdir)
   * - Existence checks
   * - Metadata inspection
   * - Relationship traversal
   *
   * Performance: 76-81% faster, 95% less bandwidth, 87% less memory
   * - Metadata-only: 10ms, 300 bytes
   * - Full entity: 43ms, 6KB
   *
   * @param id - Entity ID
   * @param metadata - Metadata from storage.getNounMetadata()
   * @returns Entity with stub vector (Float32Array(0))
   *
   */
  private async convertMetadataToEntity(id: string, metadata: any): Promise<Entity<T>> {
    // Metadata-only entity (no vector loading)
    // This is 76-81% faster for operations that don't need semantic similarity

    // Canonical reserved/custom split — same single source of truth as every
    // other read path (see src/types/reservedFields.ts).
    const { reserved, custom } = splitNounMetadataRecord(metadata)

    const entity: Entity<T> = {
      id,
      vector: [],  // Stub vector (empty array - vectors not loaded for metadata-only)
      type: (reserved.noun as NounType) || NounType.Thing,
      subtype: reserved.subtype as string | undefined,
      visibility: reserved.visibility as EntityVisibility | undefined,

      // Standard fields from metadata
      confidence: reserved.confidence as number | undefined,
      weight: reserved.weight as number | undefined,
      createdAt: (reserved.createdAt as number) || Date.now(),
      updatedAt: (reserved.updatedAt as number) || Date.now(),
      service: reserved.service as string | undefined,
      data: reserved.data,
      createdBy: reserved.createdBy as Entity<T>['createdBy'],
      // 7.31.0 — surface revision counter (defaults to 1 for pre-7.31.0 entities)
      _rev: typeof reserved._rev === 'number' ? reserved._rev : 1,

      // Custom user fields (standard fields removed, only custom remain)
      metadata: custom as T
    }

    return entity
  }


  /**
   * Update an existing entity
   *
   * Merges metadata by default — new fields are added, existing fields are overwritten,
   * and omitted fields are preserved. Set `merge: false` to replace metadata entirely.
   * If `data` is provided, the entity is re-embedded and re-indexed in HNSW.
   *
   * **Data vs Metadata:**
   * - `data`: Content used for vector embeddings (searchable via semantic similarity / HNSW).
   *   NOT queryable via `where` filters. Pass a string for text search, or any value for storage.
   * - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where` filters
   *   in `find()`. Put anything you want to filter/query on here.
   *
   * @param params - Update parameters
   * @param params.id - UUID of the entity to update (required)
   * @param params.data - New content to re-embed (triggers HNSW re-indexing)
   * @param params.type - Change entity type classification
   * @param params.metadata - Metadata fields to merge (or replace if merge=false)
   * @param params.merge - If true (default), merges metadata; if false, replaces it entirely
   * @param params.vector - Pre-computed vector (skips embedding)
   * @param params.confidence - Update type classification confidence (0-1)
   * @param params.weight - Update entity importance/salience (0-1)
   *
   * @example Update metadata (merge by default)
   * ```typescript
   * await brain.update({
   *   id: entityId,
   *   metadata: { status: 'reviewed', rating: 4.5 }
   *   // Existing metadata fields preserved, only status and rating changed
   * })
   * ```
   *
   * @example Update data (re-embeds and re-indexes)
   * ```typescript
   * await brain.update({
   *   id: entityId,
   *   data: 'Updated description of the concept'
   *   // Vector is recomputed, HNSW index updated
   * })
   * ```
   *
   * @example Replace metadata entirely
   * ```typescript
   * await brain.update({
   *   id: entityId,
   *   metadata: { onlyThisField: true },
   *   merge: false  // All previous metadata removed
   * })
   * ```
   */
  async update(params: UpdateParams<T>): Promise<void> {
    this.assertWritable('update')
    await this.ensureInitialized()

    // Zero-config validation (static import for performance)
    validateUpdateParams(params)

    // Id normalization (8.0): resolve a natural key to the canonical UUID add()
    // stored, so update() targets the same entity. A real UUID passes through.
    params = { ...params, id: resolveEntityId(params.id) }

    // Reserved fields arriving via the metadata patch are remapped to their
    // canonical top-level location, mirroring add()'s lift. Without this the
    // patch value survived the merge but was then clobbered by the

    // Tracked-field vocabulary enforcement (Layer 2). Same as add() — the
    // metadata bag carries fields registered via trackField(), and subtype is
    // a tracked top-level candidate.
    this.enforceTrackedFieldValues(params.metadata as Record<string, unknown> | undefined, 'metadata')
    if (params.subtype !== undefined) {
      this.enforceTrackedFieldValues(
        { subtype: params.subtype } as Record<string, unknown>,
        'top-level'
      )
    }

    // Subtype pairing enforcement on update (7.30.0). The effective NounType after
    // the update is `params.type ?? existing.type`; we look it up if needed.
    if (params.subtype !== undefined || params.type !== undefined) {
      const existing = await this.get(params.id)
      const effectiveType = params.type ?? existing?.type
      const effectiveSubtype = params.subtype !== undefined ? params.subtype : existing?.subtype
      const effectiveMetadata = params.metadata ?? existing?.metadata
      this.enforceSubtypeOnAdd('update', effectiveType, effectiveSubtype, effectiveMetadata)
    }

    // Get existing entity with vectors (fix for regression)
      // We need includeVectors: true because:
      // 1. SaveNounOperation requires the vector
      // 2. HNSW reindexing operations need the original vector
      const existing = await this.get(params.id, { includeVectors: true })
      if (!existing) {
        throw new EntityNotFoundError(params.id)
      }

      // ifRev (7.31.0) — optimistic concurrency. If caller supplied ifRev, the persisted
      // _rev must match exactly. `_rev` is reserved: every read path surfaces it ONLY
      // top-level (entities without one are read as rev 1), so that is the one place
      // to look. This check is purely a FAST-FAIL (avoids paying the embedding
      // cost on an obviously stale expectation) — the authoritative check runs
      // in the commit precondition below, under the commit mutex, where it is
      // atomic with the apply. Concurrent same-rev updates all pass HERE but
      // exactly one survives THERE.
      const currentRev = typeof existing._rev === 'number' ? existing._rev : 1
      if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) {
        throw new RevisionConflictError(params.id, params.ifRev, currentRev)
      }

      // Resolve the updated vector: an explicit `vector` always wins (the
      // UpdateParams contract — a new pre-computed vector, with or without
      // new `data`); otherwise new `data` re-embeds; otherwise the existing
      // vector is kept. Any vector change re-indexes HNSW below.
      let vector = existing.vector
      // 'data' is a real new value whenever it's not null/undefined — an
      // empty string ('') is legitimate content (e.g. truncating a file to
      // empty via overwrite), matching validateUpdateParams's absent-vs-empty
      // distinction. Using `Boolean(params.data)` here would treat '' as "no
      // new data", silently skipping BOTH the deferred marker and the eager
      // re-embed below — a stale vector left behind with no path to ever
      // correct itself (a quiet loss, not the deferred-but-eventually-
      // correct flicker the deferEmbedding contract promises).
      const rawHasNewData = params.data !== undefined && params.data !== null
      // NO RE-EMBED ON UNCHANGED DATA: a write carrying the row's CURRENT data
      // is not a data change — no re-embed, no deferred landing, no vector
      // rewrite. A host heartbeat re-writing an unchanged row every few
      // seconds fed a live index-row loop on a production store (each
      // "change" landed a vector); the amplifier dies here regardless of how
      // often the host writes.
      const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data)
      const hasNewData = rawHasNewData && !dataUnchanged

      // THE ZERO-NORM LAW (canonical write side) — see add()'s matching
      // comment: an explicit REAL all-zero vector is not a vector. Normalize
      // to the sanctioned "unvectored" `[]` shape BEFORE the dimension
      // check, the unvector-door decision below, and the index ops ever see
      // it — a local copy; `params.vector` itself is never mutated.
      let explicitVector = params.vector
      if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) {
        prodLog.warn(
          `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` +
            `a zero-norm vector is not a vector; persisted unvectored ([]) instead.`
        )
        explicitVector = []
      }

      // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an
      // explicit `vector: []`, or a real all-zero vector just normalized
      // above) is an instruction to remove the vector NOW — never "please
      // embed". `validateUpdateParams` already refuses combining it with
      // `deferEmbedding: true` (an empty array is truthy, so that guard
      // fires unconditionally on any explicit `vector`). Idempotent on an
      // already-unvectored row: the ledger decrement near the end of this
      // method is gated on the PRIOR vector actually having been real.
      const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0

      // MT5 deferred re-embedding: the OLD vector keeps serving semantic
      // search — stale-but-present, never absent (the flicker law) — until
      // the background worker embeds the new data and swaps it atomically.
      const deferringEmbed =
        params.deferEmbedding === true && hasNewData && !explicitVector
      if (explicitVector) {
        // A length-0 explicit vector (the unvector door) carries no
        // dimension information — exempt from the check, mirroring add()'s
        // own `vector.length > 0` gate on the dimension pin.
        if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) {
          throw new Error(
            `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}`
          )
        }
        vector = explicitVector
      } else if (hasNewData && !deferringEmbed) {
        vector = await this.embed(params.data)
      }
      // A deferred data change does NOT reindex now (the vector is unchanged;
      // the worker's atomic swap carries the real reindex later).
      const needsReindexing = Boolean(
        (hasNewData && !deferringEmbed) || params.type || explicitVector
      )

      // Always update the noun with new metadata
      const newMetadata = params.merge !== false
        ? { ...existing.metadata, ...params.metadata }
        : params.metadata || existing.metadata

      // Prepare the updated v2 nested-bag record: engine fields top-level,
      // the merged user bag nested verbatim (collider names stay the user's).
      const updatedMetadata = buildNounMetadataRecord(
        {
          data: params.data !== undefined ? params.data : existing.data,
          noun: params.type || existing.type,
          service: existing.service,
          createdAt: existing.createdAt,
          updatedAt: Date.now(),
          _rev: currentRev + 1,
          // Update confidence and weight if provided, otherwise preserve existing
          ...(params.confidence !== undefined && { confidence: params.confidence }),
          ...(params.weight !== undefined && { weight: params.weight }),
          ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }),
          ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }),
          // Update subtype if provided, otherwise preserve existing
          ...(params.subtype !== undefined && { subtype: params.subtype }),
          ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }),
          // Visibility: take the new value if provided, else preserve existing. Stored only
          // when the effective value is not 'public' (absent === public, keeps records lean).
          // A change to 'public' therefore drops the field entirely.
          ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
            visibility: params.visibility ?? existing.visibility
          })
        },
        newMetadata as Record<string, unknown>
      )

      // Build entity structure for metadata index (with top-level fields).
      // No `level`: engine plumbing never enters the indexing view (it
      // poisoned the flattened user `level` column — VENUE-BRAINY-ORDERBY-NOOP).
      const entityForIndexing = {
        id: params.id,
        vector,
        connections: new Map(),
        type: params.type || existing.type,
        subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
        ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
          visibility: params.visibility ?? existing.visibility
        }),
        confidence: params.confidence !== undefined ? params.confidence : existing.confidence,
        weight: params.weight !== undefined ? params.weight : existing.weight,
        createdAt: existing.createdAt,
        updatedAt: Date.now(),
        service: existing.service,
        data: params.data !== undefined ? params.data : existing.data,
        createdBy: existing.createdBy,
        // Only custom fields in metadata
        metadata: newMetadata
      }

      // Authoritative CAS + honest rev stamp, run by the generation store
      // UNDER the commit mutex against the just-read before-image — the one
      // point where check-and-apply is atomic (the fast-fail above can
      // interleave with concurrent writers; this cannot). Also re-stamps
      // `_rev` from the authoritative base so the counter stays monotonic
      // even for concurrent non-CAS updates. The staged operations below
      // capture `updatedMetadata` by reference, so the re-stamp lands.
      const casPrecommit = (before: CommitBeforeImages): void => {
        const beforeMeta = before.nouns.get(params.id)?.metadata as
          | { _rev?: unknown }
          | null
          | undefined
        if (!beforeMeta) {
          // A concurrent remove() deleted the entity after our read. A CAS
          // caller gets the truth; a plain update keeps its long-standing
          // last-writer-wins semantics (the staged write re-creates it).
          if (typeof params.ifRev === 'number') {
            throw new EntityNotFoundError(params.id)
          }
          return
        }
        const authoritativeRev =
          typeof beforeMeta._rev === 'number' ? beforeMeta._rev : 1
        if (typeof params.ifRev === 'number' && params.ifRev !== authoritativeRev) {
          throw new RevisionConflictError(params.id, params.ifRev, authoritativeRev)
        }
        updatedMetadata._rev = authoritativeRev + 1
      }

      // MT5: the pending marker rides the update's own commit fact (same
      // generation, one atomic append) — threaded to persistSingleOp below.
      const embedMarkers: FactMarkerRecord[] | undefined = deferringEmbed
        ? [this.enqueuePendingEmbed(params.id)]
        : undefined

      // Leg D — the unvector door clears a PENDING deferred-embed marker:
      // without this, the worker would later embed this row's current data
      // and silently re-vector it, defeating the caller's explicit "remove
      // the vector now" instruction. The clear rides THIS SAME commit fact
      // (an `embed.landed` record with an empty vector — the recovery fold
      // disarms a pending marker on ANY `embed.landed` for the id,
      // regardless of the vector it carries), so a crash between the write
      // and the in-memory clear below still recovers disarmed. Mutually
      // exclusive with `embedMarkers` above: `deferringEmbed` requires an
      // ABSENT `explicitVector`, so the two branches never both apply.
      const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id)
      const commitRecords: FactMarkerRecord[] | undefined =
        embedMarkers ?? (clearsPendingEmbed
          ? [{ type: 'embed.landed', id: params.id, vector: [] }]
          : undefined)

      // Execute atomically with transaction system, generation-stamped as one
      // immutable Model-B generation (before-image = the entity's prior state).
      await this.persistSingleOp({ nouns: [params.id] }, async (tx) => {
        // Operation 1: Update metadata FIRST (updates type cache)
        tx.addOperation(
          new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
        )

        // Operations 2-4: vector-record write + HNSW reindex — ONLY when the
        // vector side actually changed (new data/vector/type). A metadata-only
        // update must never rewrite the noun record: the record carries the
        // full vector, so an unconditional save turned every metadata touch
        // into a whole-vector rewrite + fsync — under a read-heavy consumer
        // sweep that bumps per-entity stats, this amplified into disk
        // saturation on a production deployment (SELF-ENGINE-RESTART-GRIND,
        // 2026-07-29: 5.8GB written in 40min from ~50 recalls/min).
        if (needsReindexing) {
          tx.addOperation(
            new SaveNounOperation(this.storage, {
              id: params.id,
              vector,
              connections: new Map(),
              level: 0
            })
          )
          // ONE atomic vector-index leg: the historical Remove→Add pair was
          // two separately-awaited operations — between them the row was in
          // NEITHER index (dark to semantic recall, visible to metadata
          // reads). ReplaceInVectorIndexOperation goes through the provider's
          // in-place updateItem when available (row never absent; an
          // element-wise UNCHANGED vector — the type-only-update shape that
          // flickered in production — is a pure no-op), else remove+add
          // adjacent within the single op.
          tx.addOperation(
            new ReplaceInVectorIndexOperation(this.index, params.id, existing.vector, vector, this.indexWriteGeneration)
          )
        }

        // Operation 5-6: Update metadata index (remove old, add new)
        // FIX: Include ALL indexed fields in removalMetadata (not just type)
        // Previously, only metadata + type was removed, but entityForIndexing includes:
        // confidence, weight, createdAt, updatedAt, service, data, createdBy
        // This asymmetry caused 7 fields to accumulate on EVERY update, eventually
        // making queries return 0 results (77x overcounting at scale).
        //
        // DEBUG: Log what we're removing and adding
        // console.log('[UPDATE DEBUG] existing.metadata:', JSON.stringify(existing.metadata))
        // console.log('[UPDATE DEBUG] entityForIndexing keys:', Object.keys(entityForIndexing))
        //
        // FIX: removalMetadata must MATCH entityForIndexing structure
        // entityForIndexing has: { type, confidence, ..., metadata: {...} }
        // So removalMetadata must also have: { type, confidence, ..., metadata: {...} }
        const removalMetadata = {
          type: existing.type,
          confidence: existing.confidence,
          weight: existing.weight,
          createdAt: existing.createdAt,
          updatedAt: existing.updatedAt, // CRITICAL: removes old timestamp
          service: existing.service,
          data: existing.data,
          createdBy: existing.createdBy,
          metadata: existing.metadata  // CRITICAL: keep as nested 'metadata' property!
        }
        tx.addOperation(
          new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration)
        )
        tx.addOperation(
          new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration)
        )
      }, casPrecommit, this._changeFeed.hasListeners
        ? [
            {
              kind: 'entity',
              op: 'update',
              id: params.id,
              entity: {
                id: params.id,
                type: String(entityForIndexing.type),
                ...(entityForIndexing.subtype !== undefined && {
                  subtype: String(entityForIndexing.subtype)
                }),
                metadata: (newMetadata as Record<string, unknown>) ?? {},
                ...(entityForIndexing.service !== undefined && {
                  service: String(entityForIndexing.service)
                })
              }
            }
          ]
        : undefined, commitRecords)

      // Leg D continued — the in-memory pending-embed clear runs only AFTER
      // the commit above actually succeeded (an aborted update must not
      // disarm a marker whose durable `embed.landed` twin was never
      // written).
      if (clearsPendingEmbed) {
        this.clearPendingEmbed(params.id)
        prodLog.warn(
          `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` +
            `the unvector door cleared it ('vector: []' is an explicit instruction, ` +
            `never "please embed").`
        )
      }

      // Leg D — vectored-ledger decrement for the sanctioned unvector door.
      // update()'s own metadata write goes through UpdateNounMetadataOperation
      // (isNew=false), so the saveNounMetadata(..., hasVector) seam never
      // fires here — noteVectorUnlanded is the ONLY seam, the same
      // sanctioned hook unvectorNounForRootMigration() uses. Gated on the
      // PRIOR vector having actually been real (non-empty, non-zero-norm):
      // an already-unvectored row's second call is a true no-op — no
      // decrement, matching the ledger-exactness law (never double-count,
      // never drift negative).
      if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) {
        await this.storage.noteVectorUnlanded?.(params.id)
      }

      // Aggregation hook (outside transaction — derived data). `existing` is
      // the full get() view — every reserved field top-level — and must be
      // passed whole: a subset view makes the old-side decrement miss any
      // reserved-field group (update would then double-count it).
      if (this._aggregationIndex) {
        this._aggregationIndex.onEntityUpdated(
          params.id,
          entityForIndexing,
          existing as unknown as Record<string, unknown>
        )
      }

      if (deferringEmbed) this.kickEmbedWorker()
  }

  /**
   * @description Build the metadata-index retraction operation for one id
   * (noun or verb) — the null-metadata-safe closure shared by every removal
   * leg that reaches the metadata index with a possibly-missed pre-read:
   * `remove()`'s own noun leg, its verb-cascade retractions, `unrelate()`,
   * and their `transact()`/`planTx*` mirrors (both callers add the returned
   * operation to their own batch — `tx.addOperation()` for a single-op
   * transaction, `plan.operations.push()` for a planned `transact()` batch).
   * THE NULL-METADATA SKIP IS CLOSED (a posting-leak class):
   *  - metadata present → the ordinary, provider-agnostic
   *    `RemoveFromMetadataIndexOperation` (exact per-field retraction).
   *  - metadata absent (a torn pre-read, or the row was already gone) →
   *    a provider exposing `removeEntityById` (the id-keyed contract) gets
   *    exact per-entity retraction via its reverse record; the JS index
   *    gets `removeFromIndex(id)` — safe id-keyed cleanup (deleted bitmap +
   *    id mapper; field statistics reconcile at the next rebuild/repairIndex),
   *    narrated; a native provider WITHOUT the contract is never called
   *    metadata-omitted (that path walks its value space) — the skip is
   *    tracked in the degraded set instead, narrated, so `repairIndex()`
   *    reconciles it (and this method returns `null` — no operation to add).
   *    Silence is the only thing outlawed.
   * @param id - The noun/verb id being retracted.
   * @param metadata - The pre-read metadata/entity structure, or falsy when
   *   the read missed.
   * @param context - Narration prefix identifying the caller/id, e.g.
   *   `remove(${id})` or `remove(${entityId}) cascade unrelate ${verbId}`.
   * @returns The operation to add to the caller's batch, or `null` when
   *   nothing could be done (already narrated + tracked as degraded).
   */
  /**
   * @description A JSON-safe view of a record bound for the metadata-index
   * crossing — delegates to the shared {@link jsonSafeIndexMetadata} leaf,
   * which the metadata-index transaction operations ALSO apply at execute
   * and rollback time. This plan-time wrap alone proved insufficient: it
   * returns the same reference when the record is clean, and `transact()`'s
   * delete legs share that reference with a graph-retraction op whose
   * execute-time endpoint resolution mirrors BigInt ints onto it (the full
   * aliasing story lives on the leaf module's doc).
   * @param metadata - The candidate index-metadata record.
   * @returns The same object when already JSON-safe, else a shallow copy
   *   without the BigInt-valued keys.
   */
  private static jsonSafeIndexMetadata(metadata: unknown): unknown {
    return jsonSafeIndexMetadata(metadata)
  }

  private metadataIndexRetractionOp(
    id: string,
    metadata: unknown,
    context: string
  ): Operation | null {
    if (metadata) {
      return new RemoveFromMetadataIndexOperation(
        this.metadataIndex, id, Brainy.jsonSafeIndexMetadata(metadata), this.indexWriteGeneration
      )
    }
    const prov = this.metadataIndex as unknown as {
      removeEntityById?: (id: string) => Promise<void>
      removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise<void>
    }
    if (typeof prov.removeEntityById === 'function') {
      const g = this.indexWriteGeneration
      return {
        name: 'RemoveEntityByIdTombstone',
        execute: async () => {
          await prov.removeEntityById!(id)
          return async () => {
            // Undo of an id-keyed tombstone on an absent row: nothing to
            // restore (the row had no readable metadata to re-post).
            void g
          }
        }
      }
    } else if (this.metadataIndex instanceof MetadataIndexManager) {
      const gv = this.indexWriteGeneration
      prodLog.warn(
        `[Brainy] ${context}: no metadata at delete — id-keyed index cleanup ran ` +
          `(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.`
      )
      return {
        name: 'IdKeyedIndexCleanup',
        execute: async () => {
          await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv)
          return async () => {}
        }
      }
    } else {
      this._indexDegradedIds.add(id)
      prodLog.warn(
        `[Brainy] ${context}: no metadata at delete and this provider has no id-keyed ` +
          `removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` +
          `repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` +
          `path walks the store's value space.`
      )
      return null
    }
  }

  /**
   * Remove an entity and all its relationships
   *
   * Removes the entity from all indexes (HNSW vector index, MetadataIndex,
   * GraphAdjacencyIndex) and deletes all relationships where this entity
   * is the source or target. All operations are executed atomically.
   *
   * Named `remove` to match the transact operation vocabulary
   * (`{add, update, remove, relate, unrelate}`).
   *
   * @param id - UUID of the entity to remove. Silently returns for invalid/null IDs.
   *
   * @example
   * ```typescript
   * await brain.remove(entityId)
   * const entity = await brain.get(entityId) // null
   * ```
   */
  async remove(id: string): Promise<void> {
    this.assertWritable('remove')
    // Handle invalid IDs gracefully
    if (!id || typeof id !== 'string') {
      return // Silently return for invalid IDs
    }

    await this.ensureInitialized()

    // Id normalization (8.0): resolve a natural key to the canonical UUID add()
    // stored, so remove() deletes the same entity. A real UUID passes through.
    id = resolveEntityId(id)

    // Get entity metadata and related verbs before deletion. TORN-TOLERANT:
    // a torn record must still be deletable (the delete IS the cure) — a
    // torn pre-read reads as null and the null-path below handles it loudly.
    let metadata: any = null
    let noun: any = null
    try {
      metadata = await this.storage.getNounMetadata(id)
    } catch (err) {
      if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
      prodLog.warn(`[Brainy] remove(${id}): metadata pre-read is TORN — deleting anyway; index legs run id-keyed`)
    }
    try {
      noun = await this.storage.getNoun(id)
    } catch (err) {
      if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
    }
      const verbs = await this.storage.getVerbsBySource(id)
      const targetVerbs = await this.storage.getVerbsByTarget(id)
      const allVerbs = [...verbs, ...targetVerbs]

      // Execute atomically with transaction system, generation-stamped as one
      // immutable Model-B generation covering the entity AND its cascade-deleted
      // relationships (each id's before-image = its prior state).
      await this.persistSingleOp(
        { nouns: [id], verbs: allVerbs.map((v) => v.id) },
        async (tx) => {
        // Operation 1: Remove from vector index
        if (noun) {
          tx.addOperation(
            new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)
          )
        }

        // Operation 2: Remove from metadata index (null-metadata-safe — see
        // metadataIndexRetractionOp's JSDoc for the full closure).
        {
          const retractionOp = this.metadataIndexRetractionOp(id, metadata, `remove(${id})`)
          if (retractionOp) tx.addOperation(retractionOp)
        }

        // Operation 3: Delete noun (full removal). The pre-read metadata rides
        // along so the count decrement never depends on re-reading the record
        // being removed (a null re-read must not silently skip it).
        tx.addOperation(
          new DeleteNounMetadataOperation(this.storage, id, metadata)
        )

        // Operations 4+: Delete all related verbs atomically
        for (const verb of allVerbs) {
          // Remove from graph index (endpoint ints resolved up front so a
          // rollback can re-add through the BigInt addVerb contract)
          const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
          tx.addOperation(
            new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration)
          )
          // Retract the cascaded relation's metadata-index row too — the
          // live mirror of what a rebuild would derive for this (now-gone)
          // edge (mirrors the noun leg above). The whole hydrated verb
          // (system fields top-level + the custom bag under `metadata`,
          // same shape `extractIndexableFields` reads for any entity-record
          // frame) is the before-image — every entry in `allVerbs` was
          // already successfully hydrated by the reads above, so this is
          // never metadata-omitted in practice, but the closure stays
          // defensive rather than assuming.
          {
            const cascadeRetractionOp = this.metadataIndexRetractionOp(
              verb.id, verb, `remove(${id}) cascade unrelate ${verb.id}`
            )
            if (cascadeRetractionOp) tx.addOperation(cascadeRetractionOp)
          }
          // Delete verb metadata
          tx.addOperation(
            new DeleteVerbMetadataOperation(this.storage, verb.id)
          )
        }
      },
      undefined,
      this._changeFeed.hasListeners
        ? [
            // The entity delete (payload = last state, from the pre-delete read)
            // plus one unrelate per cascade-deleted relationship.
            {
              kind: 'entity',
              op: 'remove',
              id,
              ...(metadata && {
                entity: this.entityViewFromRawRecord(id, metadata as Record<string, unknown>)
              })
            },
            ...allVerbs.map(
              (v): PendingChangeEvent => ({
                kind: 'relation',
                op: 'unrelate',
                id: v.id,
                relation: {
                  id: v.id,
                  from: v.sourceId,
                  to: v.targetId,
                  type: String(v.verb)
                }
              })
            )
          ]
        : undefined)

      // Aggregation hook (outside transaction — derived data). The view must
      // carry EVERY reserved field top-level (not a subset): a groupBy on
      // subtype/visibility/etc. otherwise decrements a nonexistent group and
      // the real count never comes down. A delete whose before-image is
      // unavailable can no longer SKIP the hook silently (the gated skip let
      // counts drift upward forever) — it flags an exact rescan, loudly.
      if (this._aggregationIndex) {
        if (metadata) {
          this._aggregationIndex.onEntityDeleted(
            id,
            this.entityForAggFromRawRecord(metadata as Record<string, unknown>)
          )
        } else {
          this._aggregationIndex.flagAllForRescan(
            `delete of ${id} carried no before-image metadata — contribution unknowable`
          )
        }
      }
  }

  // ============= RELATIONSHIP OPERATIONS =============

  // --- 8.0 u64 boundary helpers -------------------------------------------
  // UUID ↔ int conversion happens ONCE here at the coordinator boundary:
  // `getOrAssign` on writes, `getInt` on reads (undefined → the entity was
  // never mapped, i.e. it has no relations — return empty without calling the
  // provider). Provider returns convert back via `getUuid` (entities) and
  // `verbIntsToIds` + the warm cache (verbs). `Number(bigint)` narrowing is
  // lossless under the shipped EntityIdSpaceExceeded u32 guard.

  /**
   * Resolve a UUID to its entity int for a READ — `undefined` means the
   * entity was never mapped and therefore has no relations.
   */
  private graphEntityInt(uuid: string): bigint | undefined {
    const intId = this.metadataIndex.getIdMapper().getInt(uuid)
    return intId === undefined ? undefined : BigInt(intId)
  }

  /**
   * Resolve a verb's endpoint UUIDs to entity ints for a WRITE
   * (`getOrAssign`), mirroring them onto `verb.sourceInt`/`verb.targetInt`
   * (derived state — never persisted to storage JSON) before the verb is
   * handed to the graph-index provider. Accepts any verb shape carrying
   * endpoint UUIDs (`GraphVerb`, `HNSWVerbWithMetadata`, …).
   */
  private resolveVerbEndpointInts(
    verb: Pick<GraphVerb, 'sourceId' | 'targetId'> & { sourceInt?: bigint; targetInt?: bigint }
  ): { sourceInt: bigint; targetInt: bigint } {
    const idMapper = this.metadataIndex.getIdMapper()
    // Thread the write generation into any mint: a native mapper stamps the
    // assignment record with the real watermark instead of a literal 0.
    // Evaluated HERE (mint time) — at execute time inside a batch this is the
    // in-flight commit generation; at plan time it is the pre-batch watermark
    // (truthful: the mint happened before the batch committed).
    const generation = this.indexWriteGeneration()
    const sourceInt = BigInt(idMapper.getOrAssign(verb.sourceId, generation))
    const targetInt = BigInt(idMapper.getOrAssign(verb.targetId, generation))
    verb.sourceInt = sourceInt
    verb.targetInt = targetInt
    return { sourceInt, targetInt }
  }

  /**
   * Convert provider-returned entity ints back to UUIDs, dropping ints the
   * mapper no longer knows (deleted entities).
   */
  private entityIntsToUuids(entityInts: bigint[]): string[] {
    const idMapper = this.metadataIndex.getIdMapper()
    const uuids: string[] = []
    for (const entityInt of entityInts) {
      const uuid = idMapper.getUuid(Number(entityInt))
      if (uuid !== undefined) uuids.push(uuid)
    }
    return uuids
  }

  /**
   * Record one verb-int → verb-id pair in the bounded warm cache, evicting
   * the oldest entries (insertion order) past the cap.
   */
  private cacheVerbInt(verbInt: bigint, verbId: string): void {
    if (!this.verbIntWarmCache.has(verbInt) &&
        this.verbIntWarmCache.size >= Brainy.VERB_INT_WARM_CACHE_MAX) {
      // Evict oldest insertions until under cap (single eviction in the
      // common case; loop guards against future cap reductions).
      for (const oldest of this.verbIntWarmCache.keys()) {
        this.verbIntWarmCache.delete(oldest)
        if (this.verbIntWarmCache.size < Brainy.VERB_INT_WARM_CACHE_MAX) break
      }
    }
    this.verbIntWarmCache.set(verbInt, verbId)
  }

  /**
   * Resolve provider-returned verb ints to verb-id strings: warm cache first,
   * then one batched `verbIntsToIds` call for the misses (which also refills
   * the cache). Unknown ints are dropped.
   */
  private async resolveVerbIntsToIds(verbInts: bigint[]): Promise<string[]> {
    if (verbInts.length === 0) return []

    const resolved = new Array<string | undefined>(verbInts.length)
    const missIndices: number[] = []
    const missInts: bigint[] = []

    for (let i = 0; i < verbInts.length; i++) {
      const cached = this.verbIntWarmCache.get(verbInts[i])
      if (cached !== undefined) {
        resolved[i] = cached
      } else {
        missIndices.push(i)
        missInts.push(verbInts[i])
      }
    }

    if (missInts.length > 0) {
      const ids = await this.graphIndex.verbIntsToIds(missInts)
      for (let j = 0; j < missInts.length; j++) {
        const id = ids[j]
        if (id !== null) {
          resolved[missIndices[j]] = id
          this.cacheVerbInt(missInts[j], id)
        }
      }
    }

    return resolved.filter((id): id is string => id !== undefined)
  }

  /**
   * UUID-level neighbor lookup over the BigInt provider contract: resolves
   * the anchor via `getInt` (unmapped → empty), then maps returned entity
   * ints back to UUIDs. Shared by the traversal paths and the
   * TripleIntelligenceSystem adapter.
   */
  private async getNeighborUuids(
    uuid: string,
    options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number }
  ): Promise<string[]> {
    // READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
    // index read funnels through this helper, so the gate here makes
    // serve-while-not-ready UNREPRESENTABLE — a production store once acked
    // writes while every non-find() read served empty from a not-ready
    // provider for 15 minutes. A CHECK only — it never builds; throws a typed
    // NotReady error if a provider's health report says it isn't serving.
    this.ensureIndexesLoaded(['graph'])
    const entityInt = this.graphEntityInt(uuid)
    if (entityInt === undefined) return []
    const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
    return this.entityIntsToUuids(neighborInts)
  }

  /**
   * @description Hydrate the entity id-mapper from persisted state BEFORE a graph
   * rebuild. A native int-keyed adjacency resolves every verb endpoint
   * (`sourceId`/`targetId` → stable int) through this mapper, so it must reflect
   * the persisted int assignments before `graphIndex.rebuild()` runs — otherwise
   * edges resolve to stale/missing ints and are silently dropped
   * (CTX-BR-RESTORE-REBUILD). The JS mapper has no `rebuild()` and needs no
   * reload (it is re-derived by `MetadataIndex.rebuild()` via append-only
   * `getOrAssign`), so this is a no-op for it. Mirrors the ordering in
   * {@link restore}.
   */
  private async hydrateIdMapperForGraphRebuild(): Promise<void> {
    const idMapper = this.metadataIndex.getIdMapper() as { rebuild?: () => Promise<void> }
    if (typeof idMapper.rebuild === 'function') {
      await idMapper.rebuild()
    }
  }

  /**
   * @description Verify that the graph adjacency is actually LIVE before a graph read trusts
   * its result. A native graph index can load its relationship COUNT (manifest) on a cold open
   * but NOT its source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even
   * though edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would
   * serve that `[]` as if it were truth.
   *
   * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the dark-rebuild
   * failure mode this contract retires (open() alone owns building; see
   * {@link rebuildIndexesIfNeeded}). Two detection strategies, in order of honesty:
   *  - **Preferred:** {@link assessProviderHealth} — the provider's named `healthReport()` when
   *    exposed, else its sync `isReady()`. Not serving → THROW {@link GraphIndexNotReadyError}
   *    naming the reasons, immediately — no rebuild attempt.
   *  - **Fallback (providers with neither signal):** a READ-ONLY GLOBAL known-edge sample (a real
   *    persisted verb's `sourceId`, which by definition HAS an outgoing edge) — NOT any queried
   *    anchor, because brainy cannot cheaply tell "adjacency unloaded" from "this node is
   *    genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, THROW —
   *    the probe refuses loudly; it does not self-heal.
   *
   * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing to
   *   verify).
   * @throws {GraphIndexNotReadyError} when the index is not serving, or claims edges but cannot
   *   serve a known persisted edge.
   */
  private async verifyGraphAdjacencyLive(): Promise<'live'> {
    if (this._graphAdjacencyVerified) return 'live'
    // Coordinated migration LOCK (#18): while the graph provider owns a locked
    // rebuild-from-canonical, brainy must NOT judge it here — the provider owns
    // its index until it verifies-and-swaps. The data-plane lock
    // (awaitMigrationLock in ensureInitialized) already makes callers wait, so
    // this is normally unreachable mid-migration; the guard is defensive. It
    // deliberately does NOT set `_graphAdjacencyVerified`, so the real verify runs
    // once the migration clears.
    if (this.providerIsMigrating(this.graphIndex)) return 'live'
    // Re-entrancy: a fallback probe below calls getNeighbors(), which does not
    // re-enter this guard, but the short-circuit is kept defensively cheap.
    if (this._graphAdjacencyVerifying) return 'live'
    this._graphAdjacencyVerifying = true
    try {
      // ── Strategy 1: the health-report/isReady() authority — never rebuilds ──
      const assessment = assessProviderHealth(this.graphIndex)
      if (assessment.via === 'health-report' || assessment.via === 'is-ready') {
        if (assessment.readiness === 'ready') {
          this._graphAdjacencyVerified = true
          return 'live'
        }
        // A provider that is REBUILDING ITSELF gets a refusal that says so,
        // with its own progress: open deliberately did not wait for it (see
        // rebuildIndexesIfNeeded), so this door is temporarily closed and will
        // open on its own. Anything else is a broken index needing a repair.
        const rebuilding = assessProviderRebuild(this.graphIndex)
        if (rebuilding) {
          throw new GraphIndexNotReadyError(
            `Graph adjacency index is ${describeRebuildProgress(rebuilding)} and is not serving ` +
              `yet. find({ connected }), neighbors() and related() refuse rather than serve an ` +
              `empty result. The brain is open and every other family is serving; this door opens ` +
              `by itself when the provider reports serving — no action is needed.`
          )
        }
        throw new GraphIndexNotReadyError(
          `Graph adjacency index is not serving (via ${assessment.via}): ` +
            `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` +
            `related() refuse rather than serve an empty result — rebuild via ` +
            `repairIndex({ rebuild: ['graph'] }) or reopen the brain.`
        )
      }

      // ── Strategy 2: known-edge-sample probe (providers with neither signal) ─
      // READ-ONLY — refuses loudly on failure; never calls rebuild().
      const claimed = await this.graphIndex.size()
      if (!claimed || claimed <= 0) return 'live' // no edges claimed — nothing to verify

      const sample = await this.storage.getVerbs({ pagination: { limit: 1 } })
      const verb = sample.items?.[0]
      if (!verb || !verb.sourceId) {
        // no edges in storage — stale count, harmless; don't re-probe on every read.
        this._graphAdjacencyVerified = true
        return 'live'
      }

      // Resolve the known edge's source through THIS brain's id-mapper. An unmapped source means
      // the sample is not one of this brain's own edges — e.g. a shared on-disk store reused
      // across instances surfaces a foreign verb whose UUID this brain's resident mapper never
      // interned. We cannot prove a cold-unloaded adjacency from such a sample, so treat it as
      // INCONCLUSIVE: mark verified and return 'live' rather than throwing. (The honest cold-load
      // signal for native providers is Strategy 1, checked above; the JS baseline keeps its mapper
      // resident, so its OWN edges always resolve.)
      const sourceInt = this.graphEntityInt(verb.sourceId)
      if (sourceInt === undefined) {
        this._graphAdjacencyVerified = true
        return 'live'
      }

      // Ask the adjacency for ONE neighbor of the (mapped) known-edge source.
      const hasNeighbor = (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0
      if (hasNeighbor) {
        this._graphAdjacencyVerified = true
        return 'live' // adjacency is live — the common case
      }

      // INCONSISTENT: the index reports edges but a KNOWN-mapped persisted edge's source has none →
      // the adjacency did not load. Refuse loudly — never rebuild from a read.
      throw new GraphIndexNotReadyError(
        `Graph adjacency index reports ${claimed} relationship(s) but a persisted edge's source ` +
          `resolves to none — the persisted adjacency did not load. find({ connected }), ` +
          `neighbors() and related() refuse rather than serve an empty result — rebuild via ` +
          `repairIndex({ rebuild: ['graph'] }) or reopen the brain.`
      )
    } catch (err) {
      if (err instanceof GraphIndexNotReadyError) throw err
      // A transient probe failure must not break the actual query NOR be
      // masked as "no data". Allow a re-check on the next graph read and fall through.
      this._graphAdjacencyVerified = false
      if (!this.config.silent) {
        console.warn(`[Brainy] Graph adjacency consistency check skipped (transient): ${err}`)
      }
      return 'live'
    } finally {
      this._graphAdjacencyVerifying = false
    }
  }

  /**
   * @description The metadata field-index counterpart of {@link verifyGraphAdjacencyLive}.
   * On a cold open a native metadata provider can report data yet not serve its
   * `where` postings, so `find({ where })` silently returns `[]` — the exact
   * failure a downstream deployment reported (cold reads blanking filtered pages
   * after every restart).
   *
   * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the
   * dark-rebuild failure mode this contract retires (open() alone owns
   * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies:
   *  - **Preferred:** {@link assessProviderHealth} — the provider's named
   *    `healthReport()` when exposed, else its sync `isReady()`. Not serving →
   *    THROW {@link MetadataIndexNotReadyError} naming the reasons, immediately.
   *  - **Fallback (providers with neither signal):** a READ-ONLY known-value
   *    probe, run on the first FILTERED `find()`: take a KNOWN persisted entity
   *    + one of its plain field values and ask the index to resolve it. If the
   *    index does not return the known id, THROW — the probe refuses loudly;
   *    it does not self-heal. Inconclusive cases (empty store, no plain field
   *    to probe, a shared store surfacing a foreign entity) are treated as
   *    live — never a false throw. A migrating provider is skipped (it owns
   *    its locked rebuild).
   * @returns `'live'` when the index serves.
   */
  private async verifyMetadataLive(): Promise<'live'> {
    if (this._metadataVerified) return 'live'
    // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do
    // not race it. Defensive; the data-plane lock already gates callers upstream.
    if (this.providerIsMigrating(this.metadataIndex)) return 'live'
    // Re-entrancy: the fallback probe below calls filterIdsBelted(), which
    // re-enters ensureIndexesLoaded() (a cheap CHECK) but not this guard.
    if (this._metadataVerifying) return 'live'
    this._metadataVerifying = true
    try {
      // ── Strategy 1: the health-report/isReady() authority — never rebuilds ──
      const assessment = assessProviderHealth(this.metadataIndex)
      if (assessment.via === 'health-report' || assessment.via === 'is-ready') {
        if (assessment.readiness === 'ready') {
          this._metadataVerified = true
          return 'live'
        }
        const rebuilding = assessProviderRebuild(this.metadataIndex)
        if (rebuilding) {
          throw new MetadataIndexNotReadyError(
            `Metadata field index is ${describeRebuildProgress(rebuilding)} and is not serving ` +
              `yet. find({ where }) and other filtered reads refuse rather than serve an empty ` +
              `result. The brain is open and every other family is serving; this door opens by ` +
              `itself when the provider reports serving — no action is needed.`
          )
        }
        throw new MetadataIndexNotReadyError(
          `Metadata field index is not serving (via ${assessment.via}): ` +
            `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` +
            `reads refuse rather than serve an empty result — rebuild via ` +
            `repairIndex({ rebuild: ['metadata'] }) or reopen the brain.`
        )
      }

      // ── Strategy 2: known-value probe (providers with neither signal) ──────
      // READ-ONLY — refuses loudly on failure; never calls rebuild().
      // A KNOWN persisted entity + one plain field to probe. Sample a few so a
      // system-only entity (e.g. the VFS root) doesn't make every open inconclusive.
      const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } })
      let probe: { field: string; value: string | number | boolean; id: string } | null = null
      for (const noun of sample.items ?? []) {
        probe = this.pickMetadataProbe(noun as { id?: string; metadata?: Record<string, unknown> })
        if (probe) break
      }
      if (!probe) {
        // Empty store, or nothing with a plain user field to probe — inconclusive.
        this._metadataVerified = true
        return 'live'
      }
      const p = probe

      const probeServes = async (): Promise<boolean> => {
        try {
          const ids = await this.filterIdsBelted({ [p.field]: p.value })
          return ids.includes(p.id)
        } catch {
          // FIELD_NOT_INDEXED for a field a persisted entity actually holds is
          // itself the cold/broken signal — treat as not-serving.
          return false
        }
      }

      if (await probeServes()) {
        this._metadataVerified = true
        return 'live' // field postings are live — the common case
      }

      throw new MetadataIndexNotReadyError(
        `Metadata field index cannot serve a known persisted value of '${p.field}' — the field ` +
          `postings did not load. find({ where }) and other filtered reads refuse rather than ` +
          `serve an empty result — rebuild via repairIndex({ rebuild: ['metadata'] }) or reopen ` +
          `the brain.`
      )
    } catch (err) {
      if (err instanceof MetadataIndexNotReadyError) throw err
      // A transient probe failure must not break the query NOR mask as
      // "no data". Allow a re-check on the next filtered read and fall through.
      this._metadataVerified = false
      if (!this.config.silent) {
        console.warn(`[Brainy] Metadata consistency check skipped (transient): ${err}`)
      }
      return 'live'
    } finally {
      this._metadataVerifying = false
    }
  }

  /**
   * @description Choose one plain (scalar, user-written) field from an entity's
   * metadata to probe the field index with — skipping internal / system fields
   * (`__words__` text hash, VFS markers, reserved `visibility`/`subtype`/`service`,
   * the `noun` type alias, timestamps) that use different index paths, and any
   * non-scalar value. Returns `null` when the entity has no probeable field.
   */
  private pickMetadataProbe(
    noun: { id?: string; metadata?: Record<string, unknown> }
  ): { field: string; value: string | number | boolean; id: string } | null {
    const id = noun?.id
    const metadata = noun?.metadata
    if (!id || !metadata || typeof metadata !== 'object') return null
    const skip = new Set([
      'noun', 'type', 'subtype', 'service', 'visibility', 'id', 'vector',
      'isVFSEntity', 'vfsType', 'vfsPath', 'vfsName', 'createdAt', 'updatedAt'
    ])
    for (const [field, value] of Object.entries(metadata)) {
      if (field.startsWith('_') || skip.has(field)) continue
      if (value === null || value === undefined) continue
      const t = typeof value
      if (t === 'string' || t === 'number' || t === 'boolean') {
        return { field, value: value as string | number | boolean, id }
      }
    }
    return null
  }

  /**
   * @description The vector-index counterpart of {@link verifyGraphAdjacencyLive}
   * / {@link verifyMetadataLive}. On a cold open a native vector provider can
   * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded
   * its serving structure (the mmap/DiskANN graph) — so a pure semantic
   * `find({ query })` silently returns `[]`. A pure semantic query has
   * `hasFilterCriteria === false`, so the metadata guard never fires; this
   * guard closes that gap. Run one-shot on the first vector/proximity search.
   *
   * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the
   * dark-rebuild failure mode this contract retires (open() alone owns
   * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies:
   *  - **Preferred:** {@link assessProviderHealth} — the provider's named
   *    `healthReport()` when exposed, else its sync `isReady()`. Not serving →
   *    THROW {@link VectorIndexNotReadyError} naming the reasons, immediately.
   *  - **Fallback (providers with neither signal):** a READ-ONLY KNOWN
   *    persisted vector (sampled + hydrated) is searched against the index; if
   *    it does not self-match, THROW — the probe refuses loudly; it does not
   *    self-heal.
   * Inconclusive cases (empty store, no probeable vector, `size()===0` — where
   * the JS baseline is built at open) are treated as live: never a false
   * throw. A migrating provider is skipped (it owns its locked rebuild).
   * @returns `'live'` when the index serves.
   */
  private async verifyVectorLive(): Promise<'live'> {
    if (this._vectorVerified) return 'live'
    // Migration LOCK (#18): a migrating provider owns its in-place rebuild.
    if (this.providerIsMigrating(this.index)) return 'live'
    // Re-entrancy: the fallback probe below calls index.search(), which does
    // not re-enter this guard, but the short-circuit is kept defensively cheap.
    if (this._vectorVerifying) return 'live'
    this._vectorVerifying = true
    try {
      // ── Strategy 1: the health-report/isReady() authority — never rebuilds ──
      const assessment = assessProviderHealth(this.index)
      if (assessment.via === 'health-report' || assessment.via === 'is-ready') {
        if (assessment.readiness === 'ready') {
          this._vectorVerified = true
          return 'live'
        }
        const rebuilding = assessProviderRebuild(this.index)
        if (rebuilding) {
          throw new VectorIndexNotReadyError(
            `Vector index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` +
              `Semantic find({ query }) and proximity search refuse rather than serve an empty ` +
              `result. The brain is open and every other family is serving; this door opens by ` +
              `itself when the provider reports serving — no action is needed.`
          )
        }
        throw new VectorIndexNotReadyError(
          `Vector index is not serving (via ${assessment.via}): ` +
            `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` +
            `proximity search refuse rather than serve an empty result — rebuild via ` +
            `repairIndex({ rebuild: ['vector'] }) or reopen the brain.`
        )
      }

      // ── Strategy 2: known-vector probe (providers with neither signal) ─────
      // READ-ONLY — refuses loudly on failure; never calls rebuild().
      const claimed = this.index.size()
      if (!claimed || claimed <= 0) return 'live' // JS cold path is built at open

      const probe = await this.pickVectorProbe()
      if (!probe) {
        // Empty store, or nothing with a probeable vector — inconclusive.
        this._vectorVerified = true
        return 'live'
      }
      const p = probe

      // The failure mode we guard is the SILENT EMPTY result: a cold index that
      // loaded its COUNT but not its serving structure returns `[]` for a
      // known-present vector, while a warm index returns at least one hit. We
      // check for a NON-EMPTY result, NOT an exact self-match — HNSW is
      // approximate and `get()` may return a re-hydrated/normalized vector, so
      // demanding the exact self as top-1 would false-positive on a perfectly
      // healthy index (and wrongly throw).
      const hits = await this.index.search(p.vector, 1)
      void p.id // probe keyed on the vector; id retained for diagnostics only

      if (hits.length > 0) {
        this._vectorVerified = true
        return 'live' // serving structure is live — the common case
      }

      throw new VectorIndexNotReadyError(
        `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` +
          `results — the serving structure did not load. Semantic find({ query }) refuses rather ` +
          `than serve an empty result — rebuild via repairIndex({ rebuild: ['vector'] }) or ` +
          `reopen the brain.`
      )
    } catch (err) {
      if (err instanceof VectorIndexNotReadyError) throw err
      // A transient probe failure must not break the query NOR mask as
      // "no data". Allow a re-check on the next vector read and fall through.
      this._vectorVerified = false
      if (!this.config.silent) {
        console.warn(`[Brainy] Vector consistency check skipped (transient): ${err}`)
      }
      return 'live'
    } finally {
      this._vectorVerifying = false
    }
  }

  /**
   * @description Sample a KNOWN persisted noun and hydrate its vector, to probe
   * the vector index with. `get()` omits vectors by default, so this passes
   * `{ includeVectors: true }`. Samples a few (a system-only / vectorless entity
   * must not make every open inconclusive). Returns `null` when nothing has a
   * probeable vector.
   */
  private async pickVectorProbe(): Promise<{ id: string; vector: number[] } | null> {
    const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } })
    for (const noun of sample.items ?? []) {
      const id = (noun as { id?: string }).id
      if (!id) continue
      const full = await this.get(id, { includeVectors: true })
      const vector = (full as { vector?: number[] } | null)?.vector
      if (Array.isArray(vector) && vector.length > 0) {
        return { id, vector }
      }
    }
    return null
  }

  // -------------------------------------------------------------------------

  /**
   * Create a relationship (verb) between two entities
   *
   * Relationships connect entities with typed edges. Duplicate relationships
   * (same from, to, and type) are detected and return the existing ID.
   *
   * **Data vs Metadata (on relationships):**
   * - `data`: Opaque content stored on the relationship (e.g., a description or
   *   context for the edge). Overrides the auto-computed vector for this verb.
   * - `metadata`: Structured queryable fields on the edge (e.g., role, startDate).
   *
   * @param params - Parameters for creating the relationship
   * @param params.from - Source entity ID (required)
   * @param params.to - Target entity ID (required)
   * @param params.type - VerbType classification (required)
   * @param params.weight - Connection strength 0-1 (default: 1.0)
   * @param params.data - Content for the relationship (optional, overrides auto-computed vector)
   * @param params.metadata - Structured queryable fields on the edge
   * @param params.bidirectional - Create reverse edge too (default: false)
   * @param params.service - Multi-tenancy service name
   * @param params.confidence - Relationship certainty 0-1
   * @param params.evidence - Why this relationship exists
   * @returns Promise that resolves to the relationship ID. **Contract (8.0):**
   * relationship ids are always UUIDs (36-char 8-4-4-4-12 hex). Brainy
   * generates every verb id itself (`relate()` takes no custom id), so the
   * UUID shape is a guarantee, not a coincidence — graph-index providers may
   * key their verb-int interning on the raw UUID bytes for lossless
   * reverse lookup.
   *
   * @example
   * // Basic relationship creation
   * const userId = await brainy.add({
   *   data: { name: 'John', role: 'developer' },
   *   type: NounType.Person
   * })
   * const projectId = await brainy.add({
   *   data: { name: 'AI Assistant', status: 'active' },
   *   type: NounType.Thing
   * })
   *
   * const relationId = await brainy.relate({
   *   from: userId,
   *   to: projectId,
   *   type: VerbType.WorksOn
   * })
   *
   * @example
   * // Bidirectional relationships
   * const friendshipId = await brainy.relate({
   *   from: 'user-1',
   *   to: 'user-2',
   *   type: VerbType.Knows,
   *   bidirectional: true // Creates both directions automatically
   * })
   *
   * @example
   * // Weighted relationships for importance/strength
   * const collaborationId = await brainy.relate({
   *   from: 'team-lead',
   *   to: 'project-alpha',
   *   type: VerbType.LeadsOn,
   *   weight: 0.9, // High importance/strength
   *   metadata: {
   *     startDate: '2024-01-15',
   *     responsibility: 'technical leadership',
   *     hoursPerWeek: 40
   *   }
   * })
   *
   * @example
   * // Typed relationships with custom metadata
   * interface CollaborationMeta {
   *   role: string
   *   startDate: string
   *   skillLevel: number
   * }
   *
   * const brainy = new Brainy<CollaborationMeta>({ storage: 'filesystem' })
   * const relationId = await brainy.relate({
   *   from: 'developer-123',
   *   to: 'project-456',
   *   type: VerbType.WorksOn,
   *   weight: 0.85,
   *   metadata: {
   *     role: 'frontend developer',
   *     startDate: '2024-03-01',
   *     skillLevel: 8
   *   }
   * })
   *
   * @example
   * // Creating complex relationship networks
   * const entities = []
   * // Create entities
   * for (let i = 0; i < 5; i++) {
   *   const id = await brainy.add({
   *     data: { name: `Entity ${i}`, value: i * 10 },
   *     type: NounType.Thing
   *   })
   *   entities.push(id)
   * }
   *
   * // Create hierarchical relationships
   * for (let i = 0; i < entities.length - 1; i++) {
   *   await brainy.relate({
   *     from: entities[i],
   *     to: entities[i + 1],
   *     type: VerbType.DependsOn,
   *     weight: (i + 1) / entities.length
   *   })
   * }
   *
   * @example
   * // Error handling for invalid relationships
   * try {
   *   await brainy.relate({
   *     from: 'nonexistent-entity',
   *     to: 'another-entity',
   *     type: VerbType.RelatedTo
   *   })
   * } catch (error) {
   *   if (error instanceof EntityNotFoundError) {
   *     console.log(`Entity does not exist: ${error.id}`)
   *     // Handle missing entities...
   *   }
   * }
   */
  async relate(params: RelateParams<T>): Promise<string> {
    this.assertWritable('relate')
    await this.ensureInitialized()

    // Zero-config validation (static import for performance)
    validateRelateParams(params)

    // Id normalization (8.0): resolve BOTH endpoints so a caller may relate by
    // natural key on either side. Each maps to the same canonical UUID add()
    // stored; real UUIDs pass through. (The relationship's own id is an
    // engine-minted UUID — relation ids are never caller-supplied here.)
    params = { ...params, from: resolveEntityId(params.from), to: resolveEntityId(params.to) }


    // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered
    // via brain.requireSubtype() compose with the brain-wide strict-mode flag.
    // Metadata is passed so infrastructure edges (VFS containment) can bypass
    // the missing-subtype check via the `isVFSEntity` / `isVFS` marker.
    this.enforceSubtypeOnRelate('relate', params.type, params.subtype, params.metadata)

    // Verify entities exist
    const fromEntity = await this.get(params.from)
    const toEntity = await this.get(params.to)

    if (!fromEntity) {
      throw new EntityNotFoundError(params.from, `Source entity ${params.from} not found`)
    }
    if (!toEntity) {
      throw new EntityNotFoundError(params.to, `Target entity ${params.to} not found`)
    }

    // CRITICAL FIX: Check for duplicate relationships
    // This prevents infinite loops where same relationship is created repeatedly
    // Bug #1 showed incrementing verb counts (7→8→9...) indicating duplicates
    // OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan
    // 8.0 BigInt boundary: unmapped source → no existing relations to check.
    const dupSourceInt = this.graphEntityInt(params.from)
    const verbIds = dupSourceInt === undefined
      ? []
      : await this.resolveVerbIntsToIds(await this.graphIndex.getVerbIdsBySource(dupSourceInt))

    // Batch-load verbs for 5x faster duplicate checking on GCS
    // GCS: 5 verbs = 1×50ms vs 5×50ms = 250ms (5x faster)
    if (verbIds.length > 0) {
      const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds)

      for (const [verbId, verb] of verbsMap.entries()) {
        if (verb.targetId === params.to && verb.verb === params.type) {
          // Relationship already exists - return existing ID instead of creating duplicate
          return verb.id
        }
      }
    }

    // No duplicate found - proceed with creation

    // Generate ID
    const id = uuidv4()

    // Compute relationship vector (average of entities)
    const relationVector = fromEntity.vector.map(
      (v, i) => (v + toEntity.vector[i]) / 2
    )

    // Prepare verb metadata: a v2 nested-bag record — engine fields
      // top-level, the user's edge bag nested verbatim (any name is the
      // user's; the field-addressing law).
      // One timestamp for both createdAt and updatedAt so a never-updated edge reports a
      // stable updatedAt (=== createdAt) instead of a fresh Date.now() fabricated per read.
      const relateTs = Date.now()
      const verbMetadata = buildVerbMetadataRecord(
        {
          verb: params.type,
          ...(params.subtype !== undefined && { subtype: params.subtype }),
          // visibility: stored only when not 'public' (absent === public, keeps records lean)
          ...(params.visibility !== undefined &&
            params.visibility !== 'public' && { visibility: params.visibility }),
          weight: params.weight ?? 1.0,
          ...(params.confidence !== undefined && { confidence: params.confidence }),
          ...(params.service !== undefined && { service: params.service }),
          createdAt: relateTs,
          updatedAt: relateTs,
          ...(params.data !== undefined && { data: params.data })
        },
        (params.metadata as Record<string, unknown>) || {}
      )

      // Save to storage (vector and metadata separately)
      const verb: GraphVerb = {
        id,
        vector: relationVector,
        sourceId: params.from,
        targetId: params.to,
        verb: params.type,
        type: params.type,
        ...(params.subtype !== undefined && { subtype: params.subtype }),
        ...(params.visibility !== undefined &&
          params.visibility !== 'public' && { visibility: params.visibility }),
        weight: params.weight ?? 1.0,
        ...(params.confidence !== undefined && { confidence: params.confidence }),
        ...(params.service !== undefined && { service: params.service }),
        metadata: params.metadata,
        data: params.data,
        createdAt: Date.now()
      }

      // 8.0 BigInt boundary: resolve endpoint ints ONCE (getOrAssign — both
      // entities were existence-checked above) and mirror them onto the verb.
      const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)

      // Reverse-edge id reserved up front (when bidirectional) so the Model-B
      // generation's touched-verb set covers both edges this write creates.
      const reverseId = params.bidirectional ? uuidv4() : undefined

      // Execute atomically with transaction system, generation-stamped as one
      // immutable Model-B generation (before-image of each new edge = absent).
      await this.persistSingleOp({ verbs: reverseId ? [id, reverseId] : [id] }, async (tx) => {
        // Operation 1: Save verb vector data
        tx.addOperation(
          new SaveVerbOperation(this.storage, {
            id,
            vector: relationVector,
            connections: new Map(),
            verb: params.type,
            sourceId: params.from,
            targetId: params.to
          })
        )

        // Operation 2: Save verb metadata
        tx.addOperation(
          new SaveVerbMetadataOperation(this.storage, id, verbMetadata)
        )

        // Operation 3: Add to graph index for O(1) lookups
        tx.addOperation(
          new AddToGraphIndexOperation(
            this.graphIndex, verb, { sourceInt, targetInt },
            this.graphWriteGeneration,
            (verbInt) => this.cacheVerbInt(verbInt, id)
          )
        )

        // Operation 3b: Add the verb's metadata-index row, in the SAME
        // commit as the graph leg — the live mirror of what rebuild()'s
        // verb walk already derives (ADR-007 A4: one mechanism, never a
        // second hand-rolled shape). `verbMetadata` is the exact raw stored
        // record `SaveVerbMetadataOperation` above just persisted — the same
        // shape `storage.getVerbMetadata()`/rebuild() read back.
        tx.addOperation(
          new AddToMetadataIndexOperation(this.metadataIndex, id, verbMetadata, this.indexWriteGeneration)
        )

        // Create bidirectional if requested
        if (params.bidirectional && reverseId) {
          const reverseVerb: GraphVerb = {
            ...verb,
            id: reverseId,
            sourceId: params.to,
            targetId: params.from,
            // Endpoints swap, so the derived ints swap with them.
            sourceInt: targetInt,
            targetInt: sourceInt
          }

          // Operation 4: Save reverse verb vector data
          tx.addOperation(
            new SaveVerbOperation(this.storage, {
              id: reverseId,
              vector: relationVector,
              connections: new Map(),
              verb: params.type,
              sourceId: params.to,
              targetId: params.from
            })
          )

          // Operation 5: Save reverse verb metadata
          tx.addOperation(
            new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata)
          )

          // Operation 6: Add reverse relationship to graph index
          tx.addOperation(
            new AddToGraphIndexOperation(
              this.graphIndex, reverseVerb, { sourceInt: targetInt, targetInt: sourceInt },
              this.graphWriteGeneration,
              (verbInt) => this.cacheVerbInt(verbInt, reverseId)
            )
          )

          // Operation 6b: Add the reverse edge's metadata-index row (same
          // stored shape as the primary edge — SaveVerbMetadataOperation
          // above persists the same `verbMetadata` object for both).
          tx.addOperation(
            new AddToMetadataIndexOperation(this.metadataIndex, reverseId, verbMetadata, this.indexWriteGeneration)
          )
        }
      },
      undefined,
      this._changeFeed.hasListeners
        ? [
            {
              kind: 'relation',
              op: 'relate',
              id,
              relation: {
                id,
                from: params.from,
                to: params.to,
                type: String(params.type),
                ...(params.metadata && {
                  metadata: params.metadata as Record<string, unknown>
                })
              }
            },
            ...(reverseId
              ? [
                  {
                    kind: 'relation',
                    op: 'relate',
                    id: reverseId,
                    relation: {
                      id: reverseId,
                      from: params.to,
                      to: params.from,
                      type: String(params.type),
                      ...(params.metadata && {
                        metadata: params.metadata as Record<string, unknown>
                      })
                    }
                  } as PendingChangeEvent
                ]
              : [])
          ]
        : undefined)

      return id
  }

  /**
   * Delete a relationship (verb) by its ID
   *
   * Removes the relationship from the GraphAdjacencyIndex and deletes
   * the verb metadata from storage. Executed atomically.
   *
   * @param id - UUID of the relationship to delete
   *
   * @example
   * ```typescript
   * const relId = await brain.relate({
   *   from: personId, to: projectId, type: VerbType.WorksOn
   * })
   * await brain.unrelate(relId) // Relationship removed
   * ```
   */
  async unrelate(id: string): Promise<void> {
    this.assertWritable('unrelate')
    await this.ensureInitialized()

    // Get verb data before deletion for rollback
      const verb = await this.storage.getVerb(id)

      // 8.0 BigInt boundary: resolve endpoint ints before the transaction so
      // a rollback can re-add through the BigInt addVerb contract.
      const endpointInts = verb ? this.resolveVerbEndpointInts(verb) : undefined

      // Execute atomically with transaction system, generation-stamped as one
      // immutable Model-B generation (before-image = the relationship's state).
      await this.persistSingleOp({ verbs: [id] }, async (tx) => {
        // Operation 1: Remove from graph index
        if (verb && endpointInts) {
          tx.addOperation(
            new RemoveFromGraphIndexOperation(
              this.graphIndex, verb, endpointInts, this.graphWriteGeneration
            )
          )
        }

        // Operation 1b: Retract the verb's metadata-index row — the live
        // mirror of remove()'s cascade leg (null-metadata-safe; see
        // metadataIndexRetractionOp's JSDoc). Nothing to retract when the
        // pre-read found no verb (already gone / never existed).
        if (verb) {
          const retractionOp = this.metadataIndexRetractionOp(id, verb, `unrelate(${id})`)
          if (retractionOp) tx.addOperation(retractionOp)
        }

        // Operation 2: Delete verb metadata (which also deletes vector)
        tx.addOperation(
          new DeleteVerbMetadataOperation(this.storage, id)
        )
      },
      undefined,
      this._changeFeed.hasListeners && verb
        ? [
            {
              kind: 'relation',
              op: 'unrelate',
              id,
              relation: {
                id,
                from: verb.sourceId,
                to: verb.targetId,
                type: String(verb.verb),
                ...(verb.metadata && {
                  metadata: verb.metadata as Record<string, unknown>
                })
              }
            }
          ]
        : undefined)
  }

  /**
   * Update an existing relationship.
   *
   * Mirror of `update()` for relationships. Supports changing the verb type, the
   * sub-classification (`subtype`), weight/confidence, the opaque `data` payload, and
   * structured metadata (merge by default; set `merge: false` to replace).
   *
   * If `type` changes, the relationship is re-indexed in the graph adjacency
   * (`RemoveFromGraphIndex` + `AddToGraphIndex`) so traversal by verb type stays
   * consistent. The relationship ID is preserved across the type change.
   *
   * @param params - Update parameters (`id` required; at least one field to change)
   * @throws RelationNotFoundError if the relationship doesn't exist
   *
   * @example Change subtype
   * await brain.updateRelation({ id: relId, subtype: 'dotted-line' })
   *
   * @example Merge metadata
   * await brain.updateRelation({ id: relId, metadata: { startDate: '2026-Q3' } })
   *
   * @example Replace metadata entirely
   * await brain.updateRelation({ id: relId, metadata: { only: 'this' }, merge: false })
   */
  async updateRelation(params: UpdateRelationParams<T>): Promise<void> {
    this.assertWritable('updateRelation')
    await this.ensureInitialized()

    validateUpdateRelationParams(params)


    const existing = await this.storage.getVerb(params.id)
    if (!existing) {
      throw new RelationNotFoundError(params.id)
    }

    // Legacy stored shapes carried the verb type under `type` instead of the
    // canonical `verb` field — read both, canonical first.
    const existingRec: HNSWVerbWithMetadata & { type?: VerbType } = existing
    const newVerbType = params.type ?? existingRec.verb ?? existingRec.type

    // Subtype pairing enforcement on update (7.30.0). The effective verb type after
    // the update may have changed; we check against the new type and the resulting
    // subtype value (explicit param or preserved existing).
    if (params.subtype !== undefined || params.type !== undefined) {
      const effectiveSubtype = params.subtype !== undefined
        ? params.subtype
        : existingRec.subtype
      const effectiveMetadata = params.metadata ?? existingRec.metadata
      this.enforceSubtypeOnRelate('updateRelation', newVerbType, effectiveSubtype, effectiveMetadata)
    }
    const typeChanged = params.type !== undefined && params.type !== (existingRec.verb ?? existingRec.type)

    // Merge metadata (mirror of update() for nouns)
    const newMetadata = params.merge !== false
      ? { ...(existingRec.metadata || {}), ...(params.metadata || {}) }
      : params.metadata || existingRec.metadata

    // Build the updated stored record: v2 nested-bag — engine fields
    // top-level, the merged user bag nested verbatim (mirror of update()).
    const updatedWeight = params.weight ?? existingRec.weight ?? 1.0
    const updatedData =
      params.data !== undefined ? params.data : existingRec.data
    const updatedMetadata = buildVerbMetadataRecord(
      {
        verb: newVerbType,
        ...(params.subtype !== undefined
          ? { subtype: params.subtype }
          : existingRec.subtype !== undefined && { subtype: existingRec.subtype }),
        // Visibility: new value if provided, else preserve existing; stored only when the
        // effective value is not 'public' (a change to 'public' drops the field).
        ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && {
          visibility: params.visibility ?? existingRec.visibility
        }),
        weight: updatedWeight,
        ...(params.confidence !== undefined
          ? { confidence: params.confidence }
          : existingRec.confidence !== undefined && { confidence: existingRec.confidence }),
        // service/createdBy are fixed at relate() time — always carried forward
        // (omitting them here silently erased them on every updateRelation()).
        ...(existingRec.service !== undefined && { service: existingRec.service }),
        ...(existingRec.createdBy !== undefined && { createdBy: existingRec.createdBy }),
        createdAt: existingRec.createdAt,
        updatedAt: Date.now(),
        ...(updatedData !== undefined && { data: updatedData })
      },
      newMetadata as Record<string, unknown>
    )

    // Build the verb view used by the graph index — top-level fields mirror relate()'s.
    const verbForIndex: GraphVerb = {
      id: params.id,
      vector: existingRec.vector,
      sourceId: existingRec.sourceId,
      targetId: existingRec.targetId,
      verb: newVerbType,
      type: newVerbType,
      ...(params.subtype !== undefined
        ? { subtype: params.subtype }
        : existingRec.subtype !== undefined && { subtype: existingRec.subtype }),
      ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && {
        visibility: params.visibility ?? existingRec.visibility
      }),
      weight: updatedWeight,
      metadata: newMetadata,
      data: updatedData,
      createdAt: existingRec.createdAt
    }

    // 8.0 BigInt boundary: endpoints are unchanged across a type swap, so one
    // resolution serves both the remove (rollback re-add) and the re-add.
    const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined

    await this.persistSingleOp({ verbs: [params.id] }, async (tx) => {
      tx.addOperation(
        new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata)
      )

      // Re-post the verb's metadata-index row — remove the old shape, add
      // the new one, same commit (the plain pair; there is no update-op
      // capability for the metadata leg yet — see the GRAPH leg's
      // typeChanged branch just below for the capability this ISN'T:
      // that's the graph adjacency's own remove+add, keyed on the verb
      // TYPE changing; the metadata row updates on EVERY updateRelation()
      // call, since metadata/subtype/weight/etc. can all change without a
      // type change). `existing` is the pre-update hydrated verb (already
      // read above); `updatedMetadata` is the raw stored record just
      // persisted — the same shape relate()/rebuild() use to add.
      tx.addOperation(
        new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, Brainy.jsonSafeIndexMetadata(existing), this.indexWriteGeneration)
      )
      tx.addOperation(
        new AddToMetadataIndexOperation(this.metadataIndex, params.id, updatedMetadata, this.indexWriteGeneration)
      )

      // If the verb type changed, re-index in graph adjacency so traversal-by-type
      // stays consistent. The id is preserved across the swap.
      if (typeChanged && reindexInts) {
        tx.addOperation(
          new RemoveFromGraphIndexOperation(
            this.graphIndex, existing, reindexInts, this.graphWriteGeneration
          )
        )
        tx.addOperation(
          new AddToGraphIndexOperation(
            this.graphIndex, verbForIndex, reindexInts,
            this.graphWriteGeneration,
            (verbInt) => this.cacheVerbInt(verbInt, params.id)
          )
        )
      }
    },
    undefined,
    this._changeFeed.hasListeners
      ? [
          {
            kind: 'relation',
            op: 'updateRelation',
            id: params.id,
            relation: {
              id: params.id,
              from: verbForIndex.sourceId,
              to: verbForIndex.targetId,
              type: String(verbForIndex.verb ?? verbForIndex.type),
              ...(verbForIndex.metadata && {
                metadata: verbForIndex.metadata as Record<string, unknown>
              })
            }
          }
        ]
      : undefined)
  }

  /**
   * Get relationships between entities
   *
   * Supports multiple query patterns:
   * - No parameters: Returns all relationships (paginated, default limit: 100)
   * - String ID: Returns relationships from that entity (shorthand for { from: id })
   * - Parameters object: Fine-grained filtering and pagination
   *
   * @param paramsOrId - Optional string ID or parameters object
   * @returns Promise resolving to array of relationships
   *
   * Named `related` to match the `Db.related()` surface — the same call works
   * on a live brain and on a pinned `Db` view.
   *
   * @example
   * ```typescript
   * // Get all relationships (first 100)
   * const all = await brain.related()
   *
   * // Get relationships from specific entity (shorthand syntax)
   * const fromEntity = await brain.related(entityId)
   *
   * // Get relationships with filters
   * const filtered = await brain.related({
   *   type: VerbType.FriendOf,
   *   limit: 50
   * })
   *
   * // Pagination
   * const page2 = await brain.related({ offset: 100, limit: 100 })
   * ```
   *
   */
  async related(
    paramsOrId?: string | RelatedParams
  ): Promise<Relation<T>[]> {
    // Graph read: relationship traversal consults only the graph adjacency
    // family, so it waits on a graph migration but not on vector/metadata.
    await this.ensureInitialized({ needs: ['graph'] })
    await this.verifyGraphAdjacencyLive()

    // Handle string ID shorthand: related(id) -> related({ from: id })
    const rawParams = typeof paramsOrId === 'string'
      ? { from: paramsOrId }
      : (paramsOrId || {})

    // Id normalization (8.0): resolve the anchor id(s) so a caller may traverse
    // by natural key. Each maps to the canonical UUID add() stored; real UUIDs
    // pass through. Mutates a copy so the caller's params object is untouched.
    const params: RelatedParams = {
      ...rawParams,
      ...(rawParams.from !== undefined && { from: resolveEntityId(rawParams.from) }),
      ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }),
      ...(rawParams.node !== undefined && { node: resolveEntityId(rawParams.node) })
    }

    const limit = params.limit || 100
    const offset = params.offset || 0

    // `node` is the both-direction shorthand; it cannot also constrain one direction.
    if (params.node !== undefined && (params.from !== undefined || params.to !== undefined)) {
      throw new Error(
        "related(): 'node' is mutually exclusive with 'from'/'to' — pass 'node' for both-direction incident edges, or 'from'/'to' for one direction."
      )
    }

    // Production safety: warn for large unfiltered queries
    if (!params.from && !params.to && !params.node && !params.type && limit > 10000) {
      console.warn(
        `[Brainy] related(): Fetching ${limit} relationships without filters. ` +
        `Consider adding 'from', 'to', 'node', or 'type' filter for better performance.`
      )
    }

    // Shared filter (type / subtype / service / visibility). Endpoint ids
    // (`sourceId` / `targetId`) are added per-direction below.
    const filter: any = {}

    if (params.type) {
      filter.verbType = Array.isArray(params.type) ? params.type : [params.type]
    }

    if (params.subtype !== undefined) {
      filter.subtype = Array.isArray(params.subtype) ? params.subtype : [params.subtype]
    }

    if (params.service) {
      filter.service = params.service
    }

    // Visibility (8.0): exclude hidden tiers by default. Applied on the O(degree)
    // candidate set in the graph-index fast path (see baseStorage.applyVerbMetadataFilters).
    const excludedTiers = this.excludedVisibilityTiers(params)
    if (excludedTiers) {
      filter.excludeVisibility = excludedTiers
    }

    // VFS relationships are no longer filtered
    // VFS is part of the knowledge graph - users can filter explicitly if needed

    // Both-direction incident-edge query: union the node's out-edges (it is the
    // source) and in-edges (it is the target) — each an O(degree) indexed lookup —
    // then dedupe a self-loop that appears in both. Sorted by id for a stable page,
    // sliced from the merged set. Fetching the node's full incidence is the intended
    // O(degree) cost; for a very-high-degree node, deep `offset` paging is best-effort
    // (use the native edgesForNode / a graph cursor for exhaustive paging).
    if (params.node !== undefined) {
      const endLimit = offset + limit
      const [outEdges, inEdges] = await Promise.all([
        this.storage.getVerbs({
          pagination: { limit: endLimit },
          filter: { ...filter, sourceId: params.node }
        }),
        this.storage.getVerbs({
          pagination: { limit: endLimit },
          filter: { ...filter, targetId: params.node }
        })
      ])
      const byId = new Map<string, (typeof outEdges)['items'][number]>()
      for (const verb of outEdges.items) byId.set(verb.id, verb)
      for (const verb of inEdges.items) byId.set(verb.id, verb)
      const merged = [...byId.values()].sort((a, b) =>
        a.id < b.id ? -1 : a.id > b.id ? 1 : 0
      )
      return this.verbsToRelations(merged.slice(offset, offset + limit))
    }

    // Single-direction query (from / to).
    if (params.from) {
      filter.sourceId = params.from
    }

    if (params.to) {
      filter.targetId = params.to
    }

    // Fetch from storage with pagination at storage layer (efficient!)
    const result = await this.storage.getVerbs({
      pagination: {
        limit,
        offset,
        cursor: params.cursor
      },
      filter: Object.keys(filter).length > 0 ? filter : undefined
    })

    // Convert to Relation format
    return this.verbsToRelations(result.items)
  }

  // ============= GRAPH VIEWS (brain.graph.*) =============

  /**
   * @description The `brain.graph` namespace — graph-shaped reads. `subgraph`
   * extracts the multi-hop neighborhood around seed entities; the surface grows
   * with the engine (streaming `export`, `rank`, `path`, `communities`). Routes
   * to a registered native {@link GraphAccelerationProvider} when present, else
   * serves the same results from Brainy's pure-TS adjacency. Reads are live
   * ("now"); historical graph views come from `db.asOf(g).graph.*` (a later slice).
   * @returns The {@link GraphApi} bound to this brain.
   * @example
   * const view = await brain.graph.subgraph(personId, { depth: 2 })
   * // view.nodes = the 2-hop neighborhood; view.edges = the relations among them
   */
  get graph(): GraphApi<T> {
    return {
      subgraph: (selector: SubgraphSelector<T>, options?: SubgraphOptions) =>
        this.graphSubgraph(selector, options),
      export: (options?: GraphExportOptions) => this.graphExport(options),
      rank: (options?: GraphRankOptions) => this.graphRank(options),
      communities: (options?: GraphCommunitiesOptions) => this.graphCommunities(options),
      path: (from: string, to: string, options?: GraphPathOptions) =>
        this.graphPath(from, to, options)
    }
  }

  /** Resolve the optional native graph-acceleration provider (feature-detected). */
  private graphAccelerationProvider(): GraphAccelerationProvider | undefined {
    // Resolve + cache the provider INSTANCE once. Accept EITHER a ready instance OR
    // a `(storage) => provider` factory (the convention the graphIndex/metadataIndex/
    // vector providers use) — a registered factory would otherwise fail the
    // isGraphAccelerationProvider duck-test and the native path would silently never engage.
    if (this._graphAccelProvider === undefined) {
      const raw = this.pluginRegistry.getProvider<unknown>('graphAcceleration')
      let resolved: unknown = raw
      if (typeof raw === 'function' && !isGraphAccelerationProvider(raw)) {
        try {
          resolved = (raw as (storage: StorageAdapter) => unknown)(this.storage)
        } catch {
          resolved = undefined
        }
      }
      this._graphAccelProvider = isGraphAccelerationProvider(resolved) ? resolved : null
    }
    const provider = this._graphAccelProvider ?? undefined
    // Readiness gate — checked LIVE each call, never cached: the native engine sets
    // `isInitialized` false until its engine + cursor have loaded (the cold-start /
    // rebuild window). Until then, route to the pure-TS path rather than calling a
    // not-ready provider (which throws) — and re-engage the native path automatically
    // the moment `isInitialized` flips true. Gates EVERY `brain.graph.*` route
    // (subgraph/export/rank/communities/path + the query→expand fusion) at once.
    return provider && provider.isInitialized ? provider : undefined
  }

  /**
   * @description {@link GraphApi.subgraph} dispatch: native engine when present,
   * else the TS BFS fallback. The selector is an id set, a `find()` result, or a
   * query (query→expand fusion). Explicit ids are id-normalized so a caller may
   * seed by natural key.
   */
  private async graphSubgraph(
    selector: SubgraphSelector<T>,
    options?: SubgraphOptions
  ): Promise<GraphView<T>> {
    await this.ensureInitialized()
    const depth = Math.max(0, options?.depth ?? 1)
    const direction = options?.direction ?? 'both'
    const accel = this.graphAccelerationProvider()

    // Query selector (a FindParams object — not a string / array) → query→expand.
    if (selector && typeof selector === 'object' && !Array.isArray(selector)) {
      return this.graphSubgraphFromQuery(accel, selector, depth, direction, options)
    }

    // Id set: a string, an array of ids, or a `find()` result (Result[]). The
    // FindParams (query) case returned above, so the remaining union is the id forms.
    const seedIds = this.selectorToSeedIds(selector as string | string[] | Result<T>[])
    if (seedIds.length === 0) return { nodes: [], edges: [], truncated: false }
    return accel
      ? this.graphSubgraphNative(accel, this.seedIdsToInts(seedIds), depth, direction, options)
      : this.graphSubgraphFallback(seedIds, depth, direction, options)
  }

  /**
   * @description Query→expand fusion (graph #61): run the query, then expand the
   * subgraph from every match. When the native engine is present AND the query is a
   * pure metadata filter (no vector/text/proximity criteria) AND the metadata
   * provider exposes the opaque-set producer, the matched universe is forwarded to
   * `traverse` as an {@link OpaqueIdSet} with NO id materialization in TS — the
   * O(1)-crossing path. Otherwise the query is materialized via `find()` and its
   * result ids seed the traversal (native or TS fallback).
   */
  private async graphSubgraphFromQuery(
    accel: GraphAccelerationProvider | undefined,
    selector: FindParams<T>,
    depth: number,
    direction: 'in' | 'out' | 'both',
    options?: SubgraphOptions
  ): Promise<GraphView<T>> {
    // Native opaque fast path: a metadata-only universe never leaves the engine.
    if (accel && !this.hasVectorOrTextCriteria(selector)) {
      const filter = this.buildMetadataFilter(selector)
      const mip = this.metadataIndex as unknown as MetadataIndexProvider
      if (filter && typeof mip.getIdSetForFilter === 'function') {
        const universe = await mip.getIdSetForFilter(filter)
        return this.graphSubgraphNative(accel, universe, depth, direction, options)
      }
    }

    // General path: materialize the matched ids, then expand from them. The find
    // default limit (10) is too small to seed a neighborhood, so seed from ALL
    // matches up to a bounded cap unless the caller pinned an explicit limit.
    const matches = await this.find({
      ...selector,
      limit: selector.limit ?? Brainy.QUERY_SEED_CAP
    })
    const seedIds = matches.map((m) => m.id)
    if (seedIds.length === 0) return { nodes: [], edges: [], truncated: false }
    return accel
      ? this.graphSubgraphNative(accel, this.seedIdsToInts(seedIds), depth, direction, options)
      : this.graphSubgraphFallback(seedIds, depth, direction, options)
  }

  /** True when a query needs vector/text/proximity search (not a pure metadata filter). */
  private hasVectorOrTextCriteria(params: FindParams<T>): boolean {
    return Boolean(
      (params.query && params.query.trim() !== '') || params.vector || params.near
    )
  }

  /** Normalize an id-set selector (id / id[] / `find()` result) to canonical seed ids. */
  private selectorToSeedIds(selector: string | string[] | Result<T>[]): string[] {
    const arr = Array.isArray(selector) ? selector : [selector]
    return arr.map((s) => resolveEntityId(typeof s === 'string' ? s : s.id))
  }

  /** Resolve seed ids to graph entity ints, dropping any that were never mapped. */
  private seedIdsToInts(seedIds: string[]): bigint[] {
    const ints: bigint[] = []
    for (const id of seedIds) {
      const int = this.graphEntityInt(id)
      if (int !== undefined) ints.push(int)
    }
    return ints
  }

  /**
   * @description Pure-TS subgraph BFS — expands each frontier through the
   * O(degree) `related()` adjacency (visibility / type / subtype filters applied
   * there), dedupes edges, tracks hop depth, and honors `maxNodes` / `maxEdges`
   * caps. Correct at small/medium scale; the native provider is the at-scale path.
   */
  private async graphSubgraphFallback(
    seedIds: string[],
    depth: number,
    direction: 'in' | 'out' | 'both',
    options?: SubgraphOptions
  ): Promise<GraphView<T>> {
    const maxNodes = options?.maxNodes
    const maxEdges = options?.maxEdges
    const nodeDepth = new Map<string, number>()
    for (const id of seedIds) nodeDepth.set(id, 0)
    const edgesById = new Map<string, Relation<T>>()
    let truncated = false

    let frontier = [...seedIds]
    for (let d = 1; d <= depth && frontier.length > 0 && !truncated; d++) {
      const next: string[] = []
      for (const nodeId of frontier) {
        const incident = await this.incidentEdges(nodeId, direction, options)
        for (const edge of incident) {
          if (!edgesById.has(edge.id)) {
            if (maxEdges !== undefined && edgesById.size >= maxEdges) {
              truncated = true
              break
            }
            edgesById.set(edge.id, edge)
          }
          const neighbor = edge.from === nodeId ? edge.to : edge.from
          if (!nodeDepth.has(neighbor)) {
            if (maxNodes !== undefined && nodeDepth.size >= maxNodes) {
              truncated = true
              continue
            }
            nodeDepth.set(neighbor, d)
            next.push(neighbor)
          }
        }
        if (truncated) break
      }
      frontier = next
    }

    return this.buildGraphView(
      nodeDepth,
      [...edgesById.values()],
      options?.hydrateNodes !== false,
      truncated
    )
  }

  /** One frontier hop's incident edges in the requested direction (filters applied by `related()`). */
  private incidentEdges(
    nodeId: string,
    direction: 'in' | 'out' | 'both',
    options?: SubgraphOptions
  ): Promise<Relation<T>[]> {
    const shared: RelatedParams = {
      type: options?.type,
      subtype: options?.subtype,
      includeInternal: options?.includeInternal,
      includeSystem: options?.includeSystem,
      limit: options?.maxEdges ?? 100000
    }
    if (direction === 'out') return this.related({ ...shared, from: nodeId })
    if (direction === 'in') return this.related({ ...shared, to: nodeId })
    return this.related({ ...shared, node: nodeId })
  }

  /**
   * @description Native subgraph: resolve seeds to ints, call the provider's
   * `traverse`, then hydrate the columnar `Subgraph` (node ints → ids paired with
   * their depth, edge verb ints → `Relation`s) into a {@link GraphView}. The TS
   * fallback produces the same view, so Brainy CI exercises this shape via the
   * fallback; the native path itself is cross-layer-tested against the provider.
   */
  private async graphSubgraphNative(
    accel: GraphAccelerationProvider,
    seeds: bigint[] | OpaqueIdSet,
    depth: number,
    direction: 'in' | 'out' | 'both',
    options?: SubgraphOptions
  ): Promise<GraphView<T>> {
    // Resolved int seeds may be empty (no mapped entities); an OpaqueIdSet (Buffer)
    // is passed straight through — the provider owns its membership.
    if (Array.isArray(seeds) && seeds.length === 0) return { nodes: [], edges: [], truncated: false }

    const verbTypes = options?.type
      ? (Array.isArray(options.type) ? options.type : [options.type]).map((t) =>
          TypeUtils.getVerbIndex(t)
        )
      : undefined
    const excluded = this.excludedVisibilityTiers(options ?? {})
    const traverseOptions: TraverseOptions = {
      depth,
      direction,
      ...(verbTypes && { verbTypes }),
      ...(options?.subtype !== undefined && {
        subtypes: Array.isArray(options.subtype) ? options.subtype : [options.subtype]
      }),
      ...(excluded && { excludeVisibility: excluded as unknown as string[] }),
      ...(options?.maxNodes !== undefined && { maxNodes: options.maxNodes }),
      ...(options?.maxEdges !== undefined && { maxEdges: options.maxEdges }),
      includeEdges: true,
      includeDepth: true
    }

    const sub = await accel.traverse(seeds, traverseOptions)
    return this.hydrateNativeSubgraph(sub, options?.hydrateNodes !== false)
  }

  /**
   * @description Hydrate a columnar native `Subgraph` into a {@link GraphView}:
   * edge verb-ints → `Relation`s (via `verbIntsToIds` + `getVerbsBatchCached`),
   * and node ints paired with their depth (position-preserving — `entityIntsToUuids`
   * drops deleted ints, which would desync the depth column). Shared by the native
   * `subgraph` and `export` paths.
   * @param sub - The provider's columnar subgraph / cursor chunk.
   * @param hydrateNodes - Whether to batch-load each node's `type` / `subtype`.
   * @returns The hydrated {@link GraphView}.
   */
  private async hydrateNativeSubgraph(sub: Subgraph, hydrateNodes: boolean): Promise<GraphView<T>> {
    const verbIds = (await this.graphIndex.verbIntsToIds(Array.from(sub.edgeVerbInts))).filter(
      (id): id is string => id !== null
    )
    const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds)
    const edges = this.verbsToRelations([...verbsMap.values()])

    const idMapper = this.metadataIndex.getIdMapper()
    const nodeDepth = new Map<string, number>()
    for (let i = 0; i < sub.nodes.length; i++) {
      const uuid = idMapper.getUuid(Number(sub.nodes[i]))
      if (uuid !== undefined) nodeDepth.set(uuid, sub.nodeDepth ? sub.nodeDepth[i] : 0)
    }

    return this.buildGraphView(nodeDepth, edges, hydrateNodes, sub.truncated ?? false)
  }

  /**
   * @description Assemble a {@link GraphView} from discovered node depths + edges,
   * optionally hydrating each node's `type` / `subtype` in one batch read
   * (`hydrateNodes`, default `true`).
   */
  private async buildGraphView(
    nodeDepth: Map<string, number>,
    edges: Relation<T>[],
    hydrateNodes: boolean,
    truncated: boolean
  ): Promise<GraphView<T>> {
    const ids = [...nodeDepth.keys()]
    let entities: Map<string, Entity<T>> | undefined
    if (hydrateNodes && ids.length > 0) {
      entities = await this.batchGet(ids)
    }
    const nodes: GraphNode[] = ids.map((id) => {
      const entity = entities?.get(id)
      return {
        id,
        ...(entity && {
          type: entity.type,
          ...(entity.subtype !== undefined && { subtype: entity.subtype })
        }),
        depth: nodeDepth.get(id)
      }
    })
    return { nodes, edges, truncated }
  }

  /**
   * @description {@link GraphApi.export} dispatch: native snapshot-consistent
   * graph cursor when present, else the TS cursor walk over nouns + verbs. Both
   * are O(N+E) single passes (cursor pagination — no re-scan).
   */
  private async *graphExport(options?: GraphExportOptions): AsyncGenerator<GraphView<T>> {
    await this.ensureInitialized()
    const accel = this.graphAccelerationProvider()
    if (accel) {
      yield* this.graphExportNative(accel, options)
    } else {
      yield* this.graphExportFallback(options)
    }
  }

  /**
   * @description TS export: stream all nouns (node chunks) then all verbs (edge
   * chunks) via cursor pagination — O(N+E), no re-scan. Node refs carry
   * `type`/`subtype` straight from the noun records (no extra read). Hidden tiers
   * are excluded by default.
   */
  private async *graphExportFallback(options?: GraphExportOptions): AsyncGenerator<GraphView<T>> {
    const chunkSize = options?.chunkSize ?? 1000
    const excludedTiers = this.excludedVisibilityTiers(options ?? {})
    const excludedSet = excludedTiers ? new Set<string>(excludedTiers) : null

    if (options?.includeNodes !== false) {
      let cursor: string | undefined
      let offset = 0
      for (;;) {
        const page = await this.storage.getNouns({
          pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset }
        })
        const nodes: GraphNode[] = []
        for (const noun of page.items) {
          if (excludedSet && noun.visibility && excludedSet.has(noun.visibility)) continue
          nodes.push({
            id: noun.id,
            type: noun.type,
            ...(noun.subtype !== undefined && { subtype: noun.subtype })
          })
        }
        if (nodes.length > 0) yield { nodes, edges: [], truncated: false }
        if (!page.hasMore || page.items.length === 0) break
        if (page.nextCursor) cursor = page.nextCursor
        else offset += page.items.length
      }
    }

    if (options?.includeEdges !== false) {
      const filter = excludedTiers
        ? { excludeVisibility: excludedTiers as unknown as string[] }
        : undefined
      let cursor: string | undefined
      let offset = 0
      for (;;) {
        const page = await this.storage.getVerbs({
          pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset },
          filter
        })
        if (page.items.length > 0) {
          yield { nodes: [], edges: this.verbsToRelations(page.items), truncated: false }
        }
        if (!page.hasMore || page.items.length === 0) break
        if (page.nextCursor) cursor = page.nextCursor
        else offset += page.items.length
      }
    }
  }

  /**
   * @description Native export: open a snapshot-consistent graph cursor (pins a
   * generation — no dup/skip under concurrent writes), pull light chunks, and
   * hydrate each into a {@link GraphView}. The cursor is always closed (even on
   * early break / error). Cross-layer tested against the native provider; the TS
   * fallback exercises the same chunk shape in CI.
   */
  private async *graphExportNative(
    accel: GraphAccelerationProvider,
    options?: GraphExportOptions
  ): AsyncGenerator<GraphView<T>> {
    const excludedTiers = this.excludedVisibilityTiers(options ?? {})
    const handle = await accel.graphCursorOpen({
      direction: 'both',
      ...(excludedTiers && { excludeVisibility: excludedTiers as unknown as string[] }),
      projection: 'light'
    })
    try {
      for (;;) {
        const chunk = await accel.graphCursorNext(handle, options?.chunkSize ?? 1000)
        const view = await this.hydrateNativeSubgraph(chunk.subgraph, true)
        if (view.nodes.length > 0 || view.edges.length > 0) yield view
        if (chunk.done) break
      }
    } finally {
      await accel.graphCursorClose(handle)
    }
  }

  // ============= GRAPH ANALYTICS (rank / communities / path) =============

  /**
   * @description Load the whole visible graph into a dense integer adjacency for
   * the rank / communities fallbacks: a node-id list, an id→index map, and
   * `outAdj[i]` = the dense indices of node `i`'s out-edge targets (multiplicity
   * preserved). Streamed via {@link graphExport} (cursor pagination — O(N+E), no
   * re-scan), so hidden tiers are excluded the same way every other read excludes
   * them. Isolated nodes (no edges) are retained so they form singleton groups.
   */
  private async loadAnalyticsGraph(opts: {
    includeInternal?: boolean
    includeSystem?: boolean
  }): Promise<{ ids: string[]; outAdj: number[][] }> {
    const ids: string[] = []
    const index = new Map<string, number>()
    const outAdj: number[][] = []
    const idxOf = (id: string): number => {
      let i = index.get(id)
      if (i === undefined) {
        i = ids.length
        ids.push(id)
        index.set(id, i)
        outAdj.push([])
      }
      return i
    }

    for await (const chunk of this.graphExport({
      includeInternal: opts.includeInternal,
      includeSystem: opts.includeSystem
    })) {
      for (const node of chunk.nodes) idxOf(node.id)
      for (const edge of chunk.edges) {
        const s = idxOf(edge.from)
        const t = idxOf(edge.to)
        outAdj[s].push(t)
      }
    }

    return { ids, outAdj }
  }

  /**
   * @description {@link GraphApi.rank} dispatch: native ranking when a provider is
   * present, else the TS PageRank fallback.
   */
  private async graphRank(options?: GraphRankOptions): Promise<GraphRankEntry[]> {
    await this.ensureInitialized()
    const accel = this.graphAccelerationProvider()
    return accel ? this.graphRankNative(accel, options) : this.graphRankFallback(options)
  }

  /** TS PageRank over the dense visible adjacency; sorted descending, `topK`-capped. */
  private async graphRankFallback(options?: GraphRankOptions): Promise<GraphRankEntry[]> {
    const { ids, outAdj } = await this.loadAnalyticsGraph(options ?? {})
    if (ids.length === 0) return []
    const scores = pageRank(outAdj)
    const entries: GraphRankEntry[] = ids.map((id, i) => ({ id, score: scores[i] }))
    entries.sort((a, b) => b.score - a.score)
    return options?.topK !== undefined ? entries.slice(0, options.topK) : entries
  }

  /** Native ranking → hydrate node-ints to ids (descending order preserved). */
  private async graphRankNative(
    accel: GraphAccelerationProvider,
    options?: GraphRankOptions
  ): Promise<GraphRankEntry[]> {
    const excluded = this.excludedVisibilityTiers(options ?? {})
    const opts: RankOptions = {
      ...(options?.topK !== undefined && { topK: options.topK }),
      ...(excluded && { excludeVisibility: excluded as unknown as string[] })
    }
    const result = await accel.rank(opts)
    const idMapper = this.metadataIndex.getIdMapper()
    const entries: GraphRankEntry[] = []
    for (let i = 0; i < result.nodeInts.length; i++) {
      const uuid = idMapper.getUuid(Number(result.nodeInts[i]))
      if (uuid !== undefined) entries.push({ id: uuid, score: result.scores[i] })
    }
    return options?.topK !== undefined ? entries.slice(0, options.topK) : entries
  }

  /**
   * @description {@link GraphApi.communities} dispatch: native grouping when a
   * provider is present, else the TS connected-components fallback.
   */
  private async graphCommunities(
    options?: GraphCommunitiesOptions
  ): Promise<GraphCommunitiesResult> {
    await this.ensureInitialized()
    const accel = this.graphAccelerationProvider()
    return accel
      ? this.graphCommunitiesNative(accel, options)
      : this.graphCommunitiesFallback(options)
  }

  /**
   * @description TS grouping: weakly-connected components by default (union-find
   * over the undirected projection), or strongly-connected components when
   * `directed: true` (Tarjan). Largest group first.
   */
  private async graphCommunitiesFallback(
    options?: GraphCommunitiesOptions
  ): Promise<GraphCommunitiesResult> {
    const { ids, outAdj } = await this.loadAnalyticsGraph(options ?? {})
    if (ids.length === 0) return { groups: [], count: 0 }
    const labels = options?.directed
      ? stronglyConnectedComponents(outAdj)
      : connectedComponents(outAdj)
    return this.groupByLabel(ids, labels)
  }

  /** Collapse parallel `(id, label)` arrays into id-groups, largest first. */
  private groupByLabel(ids: string[], labels: number[]): GraphCommunitiesResult {
    const byLabel = new Map<number, string[]>()
    for (let i = 0; i < ids.length; i++) {
      const label = labels[i]
      const existing = byLabel.get(label)
      if (existing) existing.push(ids[i])
      else byLabel.set(label, [ids[i]])
    }
    const groups = [...byLabel.values()].sort((a, b) => b.length - a.length)
    return { groups, count: groups.length }
  }

  /** Native grouping → bucket node-ints by community label, hydrate to ids. */
  private async graphCommunitiesNative(
    accel: GraphAccelerationProvider,
    options?: GraphCommunitiesOptions
  ): Promise<GraphCommunitiesResult> {
    const excluded = this.excludedVisibilityTiers(options ?? {})
    const opts: CommunitiesOptions = {
      ...(options?.directed !== undefined && { directed: options.directed }),
      ...(excluded && { excludeVisibility: excluded as unknown as string[] })
    }
    const result = await accel.communities(opts)
    const idMapper = this.metadataIndex.getIdMapper()
    const byCommunity = new Map<number, string[]>()
    for (let i = 0; i < result.nodeInts.length; i++) {
      const uuid = idMapper.getUuid(Number(result.nodeInts[i]))
      if (uuid === undefined) continue
      const community = result.communityIds[i]
      const existing = byCommunity.get(community)
      if (existing) existing.push(uuid)
      else byCommunity.set(community, [uuid])
    }
    const groups = [...byCommunity.values()].sort((a, b) => b.length - a.length)
    return { groups, count: groups.length }
  }

  /**
   * @description {@link GraphApi.path} dispatch: native pathfinding when a provider
   * is present, else the TS BFS (hops) / Dijkstra (weight) fallback. Endpoints are
   * id-normalized so callers may pass natural keys.
   */
  private async graphPath(
    from: string,
    to: string,
    options?: GraphPathOptions
  ): Promise<GraphPathResult | null> {
    await this.ensureInitialized()
    const fromId = resolveEntityId(from)
    const toId = resolveEntityId(to)
    const accel = this.graphAccelerationProvider()
    return accel
      ? this.graphPathNative(accel, fromId, toId, options)
      : this.graphPathFallback(fromId, toId, options)
  }

  /**
   * @description On-demand TS pathfinding — expands frontiers through the O(degree)
   * `related()` adjacency (visibility / type filters applied there) rather than
   * loading the whole graph, so a short path terminates early. BFS for fewest hops
   * (default); Dijkstra (min-heap) for least summed edge weight (`by: 'weight'` —
   * each edge costs its stored `weight`, the 0–1 connection strength, default 1.0).
   */
  private async graphPathFallback(
    fromId: string,
    toId: string,
    options?: GraphPathOptions
  ): Promise<GraphPathResult | null> {
    if (fromId === toId) return { nodes: [fromId], relationships: [], cost: 0 }

    const direction = options?.direction ?? 'both'
    const maxDepth = options?.maxDepth ?? Number.POSITIVE_INFINITY
    const subOptions: SubgraphOptions = {
      ...(options?.type !== undefined && { type: options.type }),
      ...(options?.includeInternal !== undefined && { includeInternal: options.includeInternal }),
      ...(options?.includeSystem !== undefined && { includeSystem: options.includeSystem })
    }
    const pred = new Map<string, { prev: string; edgeId: string }>()

    if (options?.by === 'weight') {
      const dist = new Map<string, number>([[fromId, 0]])
      const hops = new Map<string, number>([[fromId, 0]])
      const settled = new Set<string>()
      const heap = new MinHeap<string>()
      heap.push(fromId, 0)
      while (heap.size > 0) {
        const node = heap.pop() as string
        if (settled.has(node)) continue
        settled.add(node)
        if (node === toId) break
        const depth = hops.get(node) as number
        if (depth >= maxDepth) continue
        const baseCost = dist.get(node) as number
        const edges = await this.incidentEdges(node, direction, subOptions)
        for (const edge of edges) {
          const neighbor = edge.from === node ? edge.to : edge.from
          if (settled.has(neighbor)) continue
          // Cost = the edge's stored weight (0–1 connection strength, default 1.0).
          // Non-negative by construction, so Dijkstra stays valid.
          const weight = edge.weight ?? 1
          const candidate = baseCost + weight
          if (!dist.has(neighbor) || candidate < (dist.get(neighbor) as number)) {
            dist.set(neighbor, candidate)
            hops.set(neighbor, depth + 1)
            pred.set(neighbor, { prev: node, edgeId: edge.id })
            heap.push(neighbor, candidate)
          }
        }
      }
      if (!settled.has(toId)) return null
      return this.reconstructPath(fromId, toId, pred, dist.get(toId) as number)
    }

    // Fewest hops — breadth-first, each edge cost 1.
    const visited = new Set<string>([fromId])
    const queue: Array<{ id: string; depth: number }> = [{ id: fromId, depth: 0 }]
    let head = 0
    while (head < queue.length) {
      const { id, depth } = queue[head++]
      if (depth >= maxDepth) continue
      const edges = await this.incidentEdges(id, direction, subOptions)
      for (const edge of edges) {
        const neighbor = edge.from === id ? edge.to : edge.from
        if (visited.has(neighbor)) continue
        visited.add(neighbor)
        pred.set(neighbor, { prev: id, edgeId: edge.id })
        if (neighbor === toId) return this.reconstructPath(fromId, toId, pred, depth + 1)
        queue.push({ id: neighbor, depth: depth + 1 })
      }
    }
    return null
  }

  /** Walk predecessor links from `toId` back to `fromId` into a forward route. */
  private reconstructPath(
    fromId: string,
    toId: string,
    pred: Map<string, { prev: string; edgeId: string }>,
    cost: number
  ): GraphPathResult {
    const nodes: string[] = []
    const relationships: string[] = []
    let cursor = toId
    while (cursor !== fromId) {
      nodes.push(cursor)
      const step = pred.get(cursor) as { prev: string; edgeId: string }
      relationships.push(step.edgeId)
      cursor = step.prev
    }
    nodes.push(fromId)
    nodes.reverse()
    relationships.reverse()
    return { nodes, relationships, cost }
  }

  /** Native pathfinding → resolve endpoints to ints, hydrate the returned route. */
  private async graphPathNative(
    accel: GraphAccelerationProvider,
    fromId: string,
    toId: string,
    options?: GraphPathOptions
  ): Promise<GraphPathResult | null> {
    const fromInt = this.graphEntityInt(fromId)
    const toInt = this.graphEntityInt(toId)
    if (fromInt === undefined || toInt === undefined) return null

    const verbTypes = options?.type
      ? (Array.isArray(options.type) ? options.type : [options.type]).map((t) =>
          TypeUtils.getVerbIndex(t)
        )
      : undefined
    const excluded = this.excludedVisibilityTiers(options ?? {})
    const pathOptions: PathOptions = {
      ...(options?.direction !== undefined && { direction: options.direction }),
      ...(options?.by !== undefined && { by: options.by }),
      ...(verbTypes && { verbTypes }),
      ...(excluded && { excludeVisibility: excluded as unknown as string[] }),
      ...(options?.maxDepth !== undefined && { maxDepth: options.maxDepth })
    }

    const result = await accel.path(fromInt, toInt, pathOptions)
    if (!result) return null
    const nodes = this.entityIntsToUuids(Array.from(result.nodeInts))
    const relationships = (
      await this.graphIndex.verbIntsToIds(Array.from(result.edgeVerbInts))
    ).filter((id): id is string => id !== null)
    return { nodes, relationships, cost: result.cost }
  }

  // ============= SEARCH & DISCOVERY =============

  /**
   * Unified find method - supports natural language and structured queries
   * Implements Triple Intelligence with parallel search optimization
   *
   * Combines three search dimensions in one query:
   * - **Vector (semantic):** `query` string is embedded and matched via HNSW similarity
   * - **Metadata:** `where` filters query the MetadataIndex (exact/range/set operators)
   * - **Graph:** `connected` traverses relationships via GraphAdjacencyIndex
   *
   * `data` is searchable via semantic/hybrid vector search (the `query` parameter).
   * `metadata` is searchable via structured `where` filters.
   * `type` in FindParams is an alias for filtering by `where.noun` (entity type).
   *
   * @param query - Natural language string or structured FindParams object
   * @returns Promise that resolves to array of search results with scores
   *
   * **Result Structure:**
   * Each result includes flattened entity fields for convenient access:
   * - `metadata`, `type`, `data` - Direct access (flattened from entity)
   * - `confidence`, `weight` - Entity confidence/importance (if set)
   * - `entity` - Full Entity object (backward compatible)
   * - `score` - Search relevance score (0-1)
   *
   * @example
   * // Natural language queries (most common)
   * const results = await brainy.find('users who work on AI projects')
   * const docs = await brainy.find('documents about machine learning')
   * const code = await brainy.find('JavaScript functions for data processing')
   *
   * @example
   * // Structured queries with filtering
   * const results = await brainy.find({
   *   query: 'artificial intelligence',
   *   type: NounType.Document,
   *   limit: 5,
   *   where: {
   *     status: 'published',
   *     author: 'expert'
   *   }
   * })
   *
   * // Access flattened fields directly
   * for (const result of results) {
   *   console.log(`Score: ${result.score}`)
   *   console.log(`Type: ${result.type}`)           // Flattened!
   *   console.log(`Metadata:`, result.metadata)     // Flattened!
   *   console.log(`Confidence: ${result.confidence ?? 'N/A'}`)  // Flattened!
   *   console.log(`Weight: ${result.weight ?? 'N/A'}`)          // Flattened!
   * }
   *
   * // Backward compatible: Nested access still works
   * console.log(result.entity.data)  // Also works
   *
   * @example
   * // Metadata-only filtering (no vector search)
   * const activeUsers = await brainy.find({
   *   type: NounType.Person,
   *   where: {
   *     status: 'active',
   *     department: 'engineering'
   *   },
   *   service: 'user-management'
   * })
   *
   * @example
   * // Vector similarity search with custom vectors
   * const queryVector = await brainy.embed('machine learning algorithms')
   * const similar = await brainy.find({
   *   vector: queryVector,
   *   limit: 10,
   *   type: [NounType.Document, NounType.Thing]
   * })
   *
   * @example
   * // Proximity search (find entities similar to existing ones)
   * const relatedContent = await brainy.find({
   *   near: 'document-123', // Find entities similar to this one
   *   limit: 8,
   *   where: {
   *     published: true
   *   }
   * })
   *
   * @example
   * // Pagination for large result sets
   * const firstPage = await brainy.find({
   *   query: 'research papers',
   *   limit: 20,
   *   offset: 0
   * })
   *
   * const secondPage = await brainy.find({
   *   query: 'research papers',
   *   limit: 20,
   *   offset: 20
   * })
   *
   * @example
   * // Complex search with multiple criteria
   * const results = await brainy.find({
   *   query: 'machine learning models',
   *   type: [NounType.Thing, NounType.Document],
   *   where: {
   *     accuracy: { $gte: 0.9 }, // Metadata filtering
   *     framework: { $in: ['tensorflow', 'pytorch'] }
   *   },
   *   service: 'ml-pipeline',
   *   limit: 15
   * })
   *
   * @example
   * // Empty query returns all entities (paginated)
   * const allEntities = await brainy.find({
   *   limit: 50,
   *   offset: 0
   * })
   *
   * @example
   * // Performance-optimized search patterns
   * // Fast metadata-only search (no vector computation)
   * const fastResults = await brainy.find({
   *   type: NounType.Person,
   *   where: { active: true },
   *   limit: 100
   * })
   *
   * // Combined vector + metadata for precision
   * const preciseResults = await brainy.find({
   *   query: 'senior developers',
   *   where: {
   *     experience: { $gte: 5 },
   *     skills: { $includes: 'javascript' }
   *   },
   *   limit: 10
   * })
   *
   * @example
   * // Error handling and result processing
   * try {
   *   const results = await brainy.find('complex query here')
   *
   *   if (results.length === 0) {
   *     console.log('No results found')
   *     return
   *   }
   *
   *   // Filter by confidence threshold
   *   const highConfidence = results.filter(r => r.score > 0.7)
   *
   *   // Sort by score (already sorted by default)
   *   const topResults = results.slice(0, 5)
   *
   *   return topResults.map(r => ({
   *     id: r.id,
   *     content: r.entity.data,
   *     confidence: r.score,
   *     metadata: r.entity.metadata
   *   }))
   * } catch (error) {
   *   console.error('Search failed:', error)
   *   return []
   * }
   *
   * @example
   * // VFS Filtering: Exclude VFS entities by default
   * // Knowledge graph queries stay clean - no VFS files in results
   * const knowledge = await brainy.find({ query: 'AI concepts' })
   * // Returns only knowledge entities, VFS files excluded
   *
   * @example
   * // VFS entities included by default
   * const everything = await brainy.find({
   *   query: 'documentation'
   * })
   * // Returns both knowledge entities AND VFS files
   *
   * @example
   * // Search only VFS files
   * const files = await brainy.find({
   *   where: { vfsType: 'file', extension: '.md' }
   * })
   *
   * @example
   * // Exclude VFS entities (if needed)
   * const concepts = await brainy.find({
   *   query: 'machine learning',
   *   excludeVFS: true  // Exclude VFS files
   * })
   */
  // ============= AGGREGATION =============

  /**
   * Define a named aggregate for incremental computation.
   *
   * Aggregate definitions persist across restarts. Once defined, all matching
   * entities (existing and future) contribute to the aggregate automatically.
   *
   * @param def - Aggregate definition (name, source filter, groupBy, metrics)
   *
   * @example
   * ```typescript
   * brain.defineAggregate({
   *   name: 'monthly_spending',
   *   source: { type: NounType.Event, where: { domain: 'financial' } },
   *   groupBy: ['category', { field: 'date', window: 'month' }],
   *   metrics: {
   *     total: { op: 'sum', field: 'amount' },
   *     count: { op: 'count' },
   *     average: { op: 'avg', field: 'amount' }
   *   }
   * })
   * ```
   */
  defineAggregate(def: AggregateDefinition): void {
    this.ensureAggregationIndex()
    this._aggregationIndex!.defineAggregate(def)
  }

  /**
   * Register a field for cardinality + per-NounType breakdown stats.
   *
   * Layer 2 of the subtype-and-facets primitive: a lightweight wrapper over the
   * aggregation engine that auto-defines an internal `__fieldCounts__<name>`
   * aggregate so consumers can query value frequencies without writing the
   * aggregate definition themselves. Backfill-on-define (shipped 7.23.0) means
   * existing entities are scanned on the first query, not at registration time.
   *
   * Use this for facets that don't warrant top-level promotion (e.g. `status`,
   * `source`, `role`). For sub-classification within a NounType, prefer the
   * top-level `subtype` field — it takes the standard-field fast path and has
   * its own statistics rollup.
   *
   * @param name - Field name to track. Resolves via the standard-fields-first /
   *   metadata-fallback path, so both `'subtype'` (top-level) and `'status'`
   *   (metadata) work the same way.
   * @param options - `perType` adds `noun` to the groupBy so counts are split
   *   by NounType; `values` registers a whitelist that rejects writes containing
   *   off-vocabulary values (validated in `add()`/`update()`).
   *
   * @example Track status without per-type breakdown
   * brain.trackField('status')
   * const counts = await brain.counts.byField('status')
   * // → { todo: 12, doing: 3, done: 47 }
   *
   * @example Track status with per-NounType breakdown
   * brain.trackField('status', { perType: true })
   * const taskCounts = await brain.counts.byField('status', { type: NounType.Task })
   * // → { todo: 8, doing: 2, done: 30 }
   *
   * @example Strict vocabulary
   * brain.trackField('priority', { values: ['low', 'medium', 'high'] })
   * // brain.add({ ..., metadata: { priority: 'urgent' } }) throws
   */
  trackField(
    name: string,
    options: { perType?: boolean; values?: string[] } = {}
  ): void {
    if (!name || typeof name !== 'string') {
      throw new Error('trackField: name must be a non-empty string')
    }
    const perType = options.perType === true
    const valuesSet = options.values && options.values.length > 0
      ? new Set(options.values)
      : undefined
    this._trackedFields.set(name, { perType, values: valuesSet })

    // Auto-define the backing aggregate. groupBy uses 'noun' (storage field name
    // for type) when perType is on, so the column-store key matches the persisted
    // shape and resolveEntityField doesn't need to rewrite the dimension.
    this.ensureAggregationIndex()
    const aggregateName = this.fieldCountsAggregateName(name)
    if (!this._aggregationIndex!.hasAggregate(aggregateName)) {
      this._aggregationIndex!.defineAggregate({
        name: aggregateName,
        source: {},
        groupBy: perType ? [name, 'system.type'] : [name],
        metrics: { count: { op: 'count' } }
      })
    }
  }

  /**
   * Internal aggregate name for a tracked field. Centralized so `trackField()`
   * and `counts.byField()` agree on the convention.
   */
  private fieldCountsAggregateName(name: string): string {
    // v2 suffix: the per-type dimension moved from the legacy 'noun' alias to
    // 'system.type' under the addressing law — a NEW name makes the ensure
    // block re-define and BACKFILL from canonical instead of silently serving
    // the old-dim definition (whose 'noun' key now reads user metadata and
    // would drift). The v1 rows are derived state, superseded not lost.
    return `__fieldCounts_v2__${name}`
  }

  /**
   * Register subtype enforcement for a specific NounType or VerbType.
   *
   * Two complementary mechanisms shipped together in 7.30.0:
   *
   * 1. **Per-type enforcement** (this method): mark a specific type as requiring
   *    a subtype, optionally with a fixed vocabulary. Writes targeting that
   *    type without a matching subtype throw at the boundary.
   *
   * 2. **Brain-wide strict mode**: enable via `new Brainy({ requireSubtype: true })`.
   *    Every public write path checks the strict-mode rule. See
   *    `BrainyConfig.requireSubtype`.
   *
   * Both compose: a per-type registration always applies regardless of the
   * brain-wide flag.
   *
   * @param type - The `NounType` or `VerbType` to register
   * @param options.values - Optional vocabulary whitelist (rejects off-vocab values)
   * @param options.required - When `true`, subtype is required on `add()`/`relate()`/`update()`/`updateRelation()` for this type (default: `true`)
   *
   * @example Lock down Person sub-classification
   * brain.requireSubtype(NounType.Person, {
   *   values: ['employee', 'customer', 'vendor'],
   *   required: true
   * })
   *
   * @example Lock down management relationships
   * brain.requireSubtype(VerbType.ReportsTo, {
   *   values: ['direct', 'dotted-line'],
   *   required: true
   * })
   */
  requireSubtype(
    type: NounType | VerbType,
    options: { values?: string[]; required?: boolean } = {}
  ): void {
    if (type === undefined || type === null) {
      throw new Error('requireSubtype: type must be a valid NounType or VerbType')
    }
    const required = options.required !== false
    const valuesSet = options.values && options.values.length > 0
      ? new Set(options.values)
      : undefined
    this._requiredSubtypes.set(String(type), { required, values: valuesSet })
  }

  /**
   * Resolve the per-type subtype rule for a given NounType or VerbType.
   * Returns `null` when no rule is registered for the type. Used by the
   * write-path enforcement hooks (`enforceSubtypeOnAdd`, `enforceSubtypeOnRelate`).
   */
  private getSubtypeRule(type: NounType | VerbType | undefined): { required: boolean; values?: Set<string> } | null {
    if (type === undefined || type === null) return null
    return this._requiredSubtypes.get(String(type)) ?? null
  }

  /**
   * Check whether brain-wide strict mode is on for a given type. Brain-wide
   * `requireSubtype: true` enforces on every type; the `{ except: [...] }`
   * form allows the listed types through. Used by the write-path enforcement
   * hooks for both nouns and verbs.
   */
  private brainWideStrictRequiresSubtype(type: NounType | VerbType | undefined): boolean {
    const flag = this.config.requireSubtype
    if (!flag) return false
    if (flag === true) return true
    // { except: [...] } form — strict except for listed types
    if (typeof flag === 'object' && Array.isArray(flag.except)) {
      if (type === undefined || type === null) return true
      return !flag.except.includes(type)
    }
    return false
  }

  /**
   * Whether a write should bypass subtype enforcement because it represents
   * internal Brainy infrastructure rather than user data. Currently triggers on:
   *
   * - `metadata.isVFSEntity === true` — Virtual File System root + directories
   *   + file entities. These are platform plumbing and follow their own
   *   subtype conventions (`'vfs-root'`, `'vfs-directory'`, `'vfs-file'`).
   * - `metadata.isVFS === true` — same intent; older marker.
   *
   * If user code sets these flags, they opt out of enforcement and accept the
   * responsibility. Documented as such on `AddParams.metadata` JSDoc.
   */
  private isInfrastructureWrite(metadata: unknown): boolean {
    if (!metadata || typeof metadata !== 'object') return false
    const m = metadata as Record<string, unknown>
    return m.isVFSEntity === true || m.isVFS === true
  }

  /**
   * Enforce subtype rules for a noun write (`add` / `update`).
   *
   * Walks the per-type registration AND the brain-wide strict-mode flag, and
   * throws with a descriptive message on missing or off-vocabulary values.
   * Called from `add()` / `addMany()` / `update()` before the storage write.
   *
   * Skips infrastructure writes (see `isInfrastructureWrite`) so Brainy's own
   * VFS root + directories + file entities don't get rejected when strict mode
   * is on. Vocabulary rules still apply to user-supplied subtypes — the bypass
   * only covers the missing-subtype case for internal plumbing.
   *
   * @param op - 'add' or 'update' (for error messages)
   * @param type - The NounType being written
   * @param subtype - The subtype value (or undefined)
   * @param metadata - The metadata bag (checked for infrastructure markers)
   */
  private enforceSubtypeOnAdd(
    op: 'add' | 'update',
    type: NounType | undefined,
    subtype: string | undefined,
    metadata?: unknown
  ): void {
    const rule = this.getSubtypeRule(type)
    const strict = this.brainWideStrictRequiresSubtype(type)
    const isInfra = this.isInfrastructureWrite(metadata)

    if (!rule && !strict) return

    if (subtype === undefined || subtype === null || subtype === '') {
      if ((rule?.required || strict) && !isInfra) {
        throw new Error(
          this.formatSubtypeError({
            op,
            kind: 'noun',
            typeName: String(type),
            issue: subtype === undefined ? 'undefined' : 'empty',
            rule,
            strict
          })
        )
      }
      return
    }

    if (rule?.values && !rule.values.has(subtype)) {
      throw new Error(
        this.formatSubtypeError({
          op,
          kind: 'noun',
          typeName: String(type),
          issue: 'off-vocabulary',
          offValue: subtype,
          rule,
          strict
        })
      )
    }
  }

  /**
   * Enforce subtype rules for a verb write (`relate` / `updateRelation`).
   * Mirror of `enforceSubtypeOnAdd` for relationships. Skips infrastructure
   * edges (VFS containment, etc.) — same `isVFSEntity` / `isVFS` markers.
   */
  private enforceSubtypeOnRelate(
    op: 'relate' | 'updateRelation',
    verb: VerbType | undefined,
    subtype: string | undefined,
    metadata?: unknown
  ): void {
    const rule = this.getSubtypeRule(verb)
    const strict = this.brainWideStrictRequiresSubtype(verb)
    const isInfra = this.isInfrastructureWrite(metadata)

    if (!rule && !strict) return

    if (subtype === undefined || subtype === null || subtype === '') {
      if ((rule?.required || strict) && !isInfra) {
        throw new Error(
          this.formatSubtypeError({
            op,
            kind: 'verb',
            typeName: String(verb),
            issue: subtype === undefined ? 'undefined' : 'empty',
            rule,
            strict
          })
        )
      }
      return
    }

    if (rule?.values && !rule.values.has(subtype)) {
      throw new Error(
        this.formatSubtypeError({
          op,
          kind: 'verb',
          typeName: String(verb),
          issue: 'off-vocabulary',
          offValue: subtype,
          rule,
          strict
        })
      )
    }
  }

  /**
   * Build the user-facing enforcement-error message.
   *
   * Three goals:
   *
   * 1. Diagnose: name the operation, the type, and what went wrong (missing,
   *    empty, or off-vocabulary).
   * 2. Locate: include the first non-Brainy frame from the current stack so
   *    the caller knows which line of THEIR code triggered the rejection —
   *    eliminates the "grep your repo for `brain.add`" debugging step.
   * 3. Teach: include the canonical migration recipe URL so the next consumer
   *    learns the SDK-vocabulary pattern from the error, not a postmortem.
   *
   * Added 7.30.1.
   */
  private formatSubtypeError(opts: {
    op: string
    kind: 'noun' | 'verb'
    typeName: string
    issue: 'undefined' | 'empty' | 'off-vocabulary'
    offValue?: string
    rule: { required: boolean; values?: Set<string> } | null
    strict: boolean
  }): string {
    const typeLabel = opts.kind === 'noun' ? 'NounType' : 'VerbType'
    const callSite = findCallerLocation()
    const vocab = opts.rule?.values ? Array.from(opts.rule.values).join(', ') : null

    let head: string
    if (opts.issue === 'off-vocabulary') {
      head = `${opts.op}(): ${typeLabel}.${opts.typeName} subtype '${opts.offValue}' is not in registered vocabulary [${vocab}].`
    } else {
      head = `${opts.op}(): ${typeLabel}.${opts.typeName} requires subtype but got ${opts.issue}.`
    }

    const callerLine = callSite ? `  at ${callSite}` : null

    let guidance: string
    if (vocab) {
      // The consumer (or a platform layer like the SDK) registered a specific
      // vocabulary. Tell them what to pass.
      guidance = `  Pass one of: ${vocab}.`
    } else if (opts.strict) {
      // Brain-wide strict mode is on without per-type values. Caller picks the
      // subtype string but it must be non-empty.
      guidance = '  Brain-wide strict mode (`requireSubtype: true`) is on — pass a non-empty `subtype` on every write, or add this type to `requireSubtype.except`.'
    } else {
      guidance = '  Register vocabulary via `brain.requireSubtype()` or pass a `subtype` value.'
    }

    const docLink = '  Migration recipe: https://soulcraft.com/docs/guides/subtypes-and-facets#strict-mode'

    return [head, callerLine, guidance, docLink].filter(Boolean).join('\n')
  }

  // findCallerLocation is now exported from src/utils/callerLocation.ts so
  // both the subtype enforcement errors here and the query-limit warnings in
  // paramValidation.ts can share it without re-importing brainy.ts.

  /**
   * Validate a metadata bag (or top-level field assignment) against any registered
   * value whitelists. Called from `add()`/`update()` after the standard zero-config
   * validation. Throws on the first off-vocabulary value to fail fast.
   *
   * Tracked fields with no `values` whitelist are skipped — registration alone
   * does not imply validation.
   */
  private enforceTrackedFieldValues(
    bag: Record<string, unknown> | undefined,
    bagLabel: 'metadata' | 'top-level'
  ): void {
    if (!bag || this._trackedFields.size === 0) return
    for (const [field, def] of this._trackedFields.entries()) {
      if (!def.values) continue
      if (!(field in bag)) continue
      const value = bag[field]
      if (value === undefined || value === null) continue
      const asString = typeof value === 'string' ? value : String(value)
      if (!def.values.has(asString)) {
        throw new Error(
          `trackField('${field}') rejected ${bagLabel} value '${asString}': not in registered vocabulary [${Array.from(def.values).join(', ')}]`
        )
      }
    }
  }

  /**
   * Remove a named aggregate and clean up its state.
   *
   * @param name - Name of the aggregate to remove
   */
  removeAggregate(name: string): void {
    if (this._aggregationIndex) {
      this._aggregationIndex.removeAggregate(name)
    }
  }

  /**
   * Query a named aggregate, returning the documented `AggregateResult[]` shape
   * (`{ groupKey, metrics, count }`) directly — the report-friendly view.
   *
   * `find({ aggregate })` returns the same data wrapped as search `Result<T>` rows (with
   * `score`/`type`/`entity`) for uniformity with the rest of `find()`; this method is the
   * first-class analytics path for dashboards and reports.
   *
   * @param name - Aggregate name (must be defined via `defineAggregate`)
   * @param params - Optional `where` (group-key filter), `having` (metric filter), `orderBy`, `order`, `limit`, `offset`
   * @returns Computed group rows
   *
   * @example
   * const rows = await brain.queryAggregate('sales_by_category', { orderBy: 'revenue', order: 'desc', limit: 10 })
   * // [{ groupKey: { category: 'food' }, metrics: { revenue: 17.5, count: 2 }, count: 2 }, ...]
   */
  async queryAggregate(
    name: string,
    params?: Omit<AggregateQueryParams, 'name'>
  ): Promise<AggregateResult[]> {
    await this.ensureInitialized()
    this.ensureAggregationIndex()
    // Persisted definitions load asynchronously — wait for them before deciding
    // the aggregate doesn't exist (an app that relies on persisted definitions
    // without re-defining at boot would otherwise race a spurious throw here).
    await this._aggregationIndex!.ready()
    if (!this._aggregationIndex!.hasAggregate(name)) {
      throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`)
    }
    await this.backfillAggregateIfNeeded(name)
    return this._aggregationIndex!.queryAggregate({ name, ...params })
  }

  /**
   * Lazily create the AggregationIndex on first use.
   * Checks for a native 'aggregation' provider from plugins.
   */
  private ensureAggregationIndex(): void {
    if (this._aggregationIndex) return

    const nativeProvider = this.pluginRegistry.getProvider<any>('aggregation')
    this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider)
    // Note: init() is async but definitions can be registered synchronously.
    // State loading happens lazily on first query.
    this._aggregationIndex.init().catch((err) => {
      // Non-fatal — definitions still work and state backfills on first query —
      // but a failed state load means aggregates read empty/stale until then, so
      // surface it loudly rather than swallow it.
      if (!this.config.silent) {
        prodLog.warn(
          `[Brainy] Aggregation index state failed to load at init ` +
          `(${(err as Error).message}). Aggregates may read empty until a ` +
          `backfill-on-query repopulates them.`
        )
      }
    })
  }

  /**
   * The visibility tiers a read should EXCLUDE, given the caller's opt-ins.
   * Default (no opts) hides both `'internal'` and `'system'`; `includeInternal`
   * unhides internal; `includeSystem` unhides system. Returns `null` when nothing
   * is excluded (both opts set) so callers can skip the filter entirely.
   *
   * @param params - The read params carrying `includeInternal` / `includeSystem`.
   * @returns The set of excluded tier strings, or `null` for "exclude nothing".
   */
  private excludedVisibilityTiers(
    params: { includeInternal?: boolean; includeSystem?: boolean }
  ): EntityVisibility[] | null {
    const excluded: EntityVisibility[] = []
    if (!params.includeInternal) excluded.push('internal')
    if (!params.includeSystem) excluded.push('system')
    return excluded.length > 0 ? excluded : null
  }

  /**
   * Resolve the set of entity/relationship ids to hide for this read, by asking
   * the metadata index for everything tagged with an excluded visibility tier.
   * Public entities carry NO stored `visibility` (absent === public), so they are
   * never in this set — exactly the entities we want to keep. Returns an empty set
   * when nothing is excluded. The result is used as a HARD candidate filter
   * (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct.
   *
   * @param params - The read params carrying the visibility opt-ins.
   * @returns A set of ids to omit from results.
   */
  private async resolveHiddenIds(
    params: { includeInternal?: boolean; includeSystem?: boolean }
  ): Promise<Set<string>> {
    const excluded = this.excludedVisibilityTiers(params)
    if (!excluded) return new Set()
    // 'system.visibility' — the engine scalar's frozen address. A bare
    // 'visibility' key would address the USER's metadata bag under the
    // field-addressing law and silently hide nothing (VFS/system entities
    // would leak into every default read).
    const ids = await this.filterIdsBelted({
      'system.visibility': excluded.length === 1 ? excluded[0] : { oneOf: excluded }
    })
    return new Set(ids)
  }

  /**
   * @description The unified Triple-Intelligence query. Composes vector similarity
   * (`query` / `vector` / `near`), metadata filtering (`type` / `subtype` / `where` /
   * `service`), and graph traversal (`connected`) into one ranked, paginated result
   * set — pass a bare string for a quick semantic search, or a {@link FindParams}
   * object to combine dimensions. Internal/system entities are hidden by default
   * (opt in with `includeInternal` / `includeSystem`); `orderBy`/`order` and
   * `limit`/`offset` apply across the composed result. Every returned row is
   * re-validated against its own predicate, so a stale or cross-bucket index entry
   * can never surface an entity that does not actually match.
   * @param query - A semantic search string, or a {@link FindParams} object combining
   *   any of `query`, `vector`, `near`, `type`, `subtype`, `where`, `connected`,
   *   `orderBy`/`order`, `limit`, `offset`.
   * @returns The matching {@link Result}s — flattened (`id`, `score`, `type`,
   *   `metadata`, `data`, …) and ranked; an empty array when nothing matches.
   * @example
   * // Semantic + metadata + graph, composed in one call:
   * const rows = await brain.find({
   *   query: 'climate policy',
   *   where: { year: { gte: 2020 } },
   *   connected: { from: orgId, via: VerbType.Authored },
   *   limit: 10
   * })
   */
  async find(query: string | FindParams<T>): Promise<Result<T>[]> {
    // Init only here — the migration gate is applied family-scoped below, once
    // the query params reveal which index families this read actually consults
    // (a string query may parse to a where / connected / vector shape). The lazy
    // loader and cold-read probes below already defer to a migrating provider.
    await this.ensureInitialized({ needs: [] })

    // READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it
    // never builds. open() already brought every provider to serving before
    // init() returned; this throws a typed NotReady error if one isn't.
    this.ensureIndexesLoaded(['metadata'])

    // Loudly flag a degraded derived index (failed init rebuild, or an
    // adopt-forward degraded commit) so a partial result is never mistaken for
    // authoritative. The streaming search() generator delegates to find(), so it
    // is covered here.
    this.warnIfReadsDegraded('find')

    // Parse natural language queries
    let params: FindParams<T> =
      typeof query === 'string' ? await this.parseNaturalQuery(query) : query

    // The vector and graph legs gate only the finds that consult them.
    const consultsVector = Boolean(
      (params.query && params.query.trim() !== '') || params.vector || params.near
    )
    if (consultsVector) this.ensureIndexesLoaded(['vector'])
    if (params.connected) this.ensureIndexesLoaded(['graph'])

    // Id normalization (8.0): resolve the graph-traversal anchor id(s) so a
    // caller may constrain by natural key. Each maps to the canonical UUID
    // add() stored; real UUIDs pass through. Done once here so every downstream
    // consumer of params.connected sees canonical ids.
    if (params.connected && (params.connected.from !== undefined || params.connected.to !== undefined)) {
      params = {
        ...params,
        connected: {
          ...params.connected,
          ...(params.connected.from !== undefined && { from: resolveEntityId(params.connected.from) }),
          ...(params.connected.to !== undefined && { to: resolveEntityId(params.connected.to) })
        }
      }
    }

    // MATCH-ALL NORMALIZATION (served-or-refused law): an empty `where: {}`
    // carries zero predicates, so it MUST route exactly like an absent `where`.
    // Left in place it reads as "filter criteria present" below, builds an
    // empty index filter, and `getIdsForFilter({})` answers `[]` by contract —
    // a silent empty on a query that semantically matches everything (worst on
    // a freshly reopened brain, where it masquerades as data loss; on the
    // vector path it short-circuits `find({ query, where: {} })` to `[]`).
    // Dropped here, ONCE, before branch selection: the query takes the
    // unfiltered match-all branch below, which serves from truth-complete
    // sources — a storage page bounded to the offset+limit window (never a
    // full walk), or the column store's top-K sort when orderBy is present.
    // Every delegating surface (Db pins via host.find, pagination.find,
    // streaming.search, subgraph query seeding) inherits this routing.
    if (params.where !== undefined && !whereConstrains(params.where)) {
      const { where: _emptyWhere, ...rest } = params
      params = rest as FindParams<T>
    }

    // Zero-config validation (static import for performance)
    validateFindParams(params)

    // Family-scoped migration gate: wait only on the index families THIS query
    // consults, so a filter or graph query served from a healthy family never
    // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once
    // params are final (after natural-language parse + connected-id resolution)
    // and before the aggregate path, which carries no gate of its own.
    await this.awaitMigrationLock(this.queryIndexFamilies(params))

    // Aggregate query path — early return when params.aggregate is set
    if (params.aggregate) {
      return this.findAggregate(params)
    }

    // Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides
    // internal + system; opt back in with includeInternal / includeSystem. Applied as a
    // hard candidate filter at every candidate-gathering point below so limit/offset stay
    // correct. Empty set when both opts are set (nothing excluded).
    const hiddenIds = await this.resolveHiddenIds(params)
    const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id)

    const startTime = Date.now()
    let result = await (async () => {
      let results: Result<T>[] = []

      // Distinguish between search criteria (need vector search) and filter criteria (metadata only)
      // Treat empty string query as no query
      const hasVectorSearchCriteria = (params.query && params.query.trim() !== '') || params.vector || params.near
      const hasFilterCriteria = params.where || params.type || params.subtype || params.service
      const hasGraphCriteria = params.connected

      // Validate the raw where clause up front: an unrecognized operator (a typo
      // like `notIn`, or any non-operator key on a field's object value) throws a
      // typed BrainyError('INVALID_QUERY') instead of silently matching nothing.
      // Done before the index/vector/graph paths so it fires even on an empty
      // result set, and before brainy injects its own internal filter fields.
      if (params.where) {
        validateWhereFilter(params.where)
      }

      // Metadata cold-read guard: before trusting a filter result, verify the
      // field index actually serves a known persisted value (one-shot per brain).
      // A cold native index that has not loaded its `where` postings self-heals
      // here (rebuild) or throws MetadataIndexNotReadyError — never a silent [].
      // The graph counterpart (verifyGraphAdjacencyLive) covers `connected`.
      if (hasFilterCriteria) {
        await this.verifyMetadataLive()
      }

      // Handle metadata-only queries (no vector search needed)
      if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) {
        // Build filter for metadata index
        let filter: any = {}
        if (params.where) {
          // Where keys pass through UNTOUCHED — the addressing law parses
          // them at the index boundary. The old where.type→noun alias is
          // dead: bare 'type' is the user's own field now.
          Object.assign(filter, params.where)
        }
        if (params.service) filter['system.service'] = params.service

        // Subtype (top-level standard field — fast path, not metadata fallback).
        // Must be assigned BEFORE the type-array expansion below so the spread
        // into each anyOf branch carries it through.
        if (params.subtype !== undefined) {
          filter['system.subtype'] = Array.isArray(params.subtype)
            ? { oneOf: params.subtype }
            : params.subtype
        }

        if (params.type) {
          const types = Array.isArray(params.type) ? params.type : [params.type]
          if (types.length === 1) {
            filter['system.type'] = types[0]
          } else {
            filter = {
              anyOf: types.map(type => ({
                'system.type': type,
                ...filter
              }))
            }
          }
        }

        // ExcludeVFS helper - ONLY exclude VFS infrastructure entities
        // Applied AFTER type filter to avoid execution order bugs
        // Excludes entities where:
        //   - vfsType is 'file' or 'directory' (VFS files/folders)
        //   - isVFSEntity is true (explicitly marked as VFS)
        // Includes extracted entities (person/concept/etc) even if they have vfsPath metadata
        if (params.excludeVFS === true) {
          // VFS infrastructure entities ALWAYS have vfsType set
          // Extracted entities do NOT have vfsType (undefined)
          filter.vfsType = { exists: false }
          // Extra safety: exclude entities explicitly marked as VFS
          filter.isVFSEntity = { ne: true }
        }

        // Apply sorting if requested, otherwise just filter
        let filteredIds: string[]
        if (params.orderBy) {
          // Get sorted IDs using production-scale sorted filtering. Bound the sort to
          // the page (offset+limit) plus the hidden over-fetch — so a broad filter +
          // orderBy returning 20 rows doesn't materialize every matching sorted id.
          const sortTopK = (params.offset || 0) + (params.limit || 10) + hiddenIds.size
          filteredIds = await this.metadataIndex.getSortedIdsForFilter(
            filter,
            params.orderBy,
            params.order || 'asc',
            sortTopK
          )
        } else {
          // Just filter without sorting. Pass a page bound so a native provider can
          // early-stop and return only the page-prefix (killing the O(N) FFI marshal
          // at billion scale). Over-fetch by the hidden count, then re-window below;
          // offset stays 0 because the visibility filter + slice happen here. The JS
          // index ignores the bound and returns all matches (behaviour unchanged).
          const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size
          filteredIds = await this.filterIdsBelted(filter, { limit: pageEnd, offset: 0 })
        }

        // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact.
        if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))

        // Paginate BEFORE loading entities (production-scale!)
        const limit = params.limit || 10
        const offset = params.offset || 0
        const pageIds = filteredIds.slice(offset, offset + limit)

        // Batch-load entities for 10x faster cloud storage performance
        // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster)
        const entitiesMap = await this.batchGet(pageIds)
        for (const id of pageIds) {
          const entity = entitiesMap.get(id)
          if (entity) {
            results.push(this.createResult(id, 1.0, entity))
          }
        }

        return results
      }

      // Handle completely empty query - return all results paginated
      if (!hasVectorSearchCriteria && !hasFilterCriteria && !hasGraphCriteria) {
        const limit = params.limit || 20
        const offset = params.offset || 0

        // Unfiltered sort: column store handles this at O(K log S) scale.
        // No per-entity storage reads, no bucketing precision loss.
        if (params.orderBy) {
          // Over-fetch by the hidden count so dropping hidden ids still fills the page.
          const k = limit + offset + hiddenIds.size
          const sortedIntIds = await this.metadataIndex.columnStore.sortTopK(
            params.orderBy,
            params.order || 'asc',
            k
          )

          // Convert int IDs (BigInt at the provider boundary) to UUIDs and
          // paginate. Number() narrowing is lossless under the u32 guard.
          const idMapper = this.metadataIndex.getIdMapper()
          let allUuids = sortedIntIds
            .map(intId => idMapper.getUuid(Number(intId)))
            .filter((uuid): uuid is string => uuid !== undefined)
          // Visibility hard filter — drop hidden ids BEFORE pagination.
          if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id))
          const pageIds = allUuids.slice(offset, offset + limit)

          const entitiesMap = await this.batchGet(pageIds)
          for (const id of pageIds) {
            const entity = entitiesMap.get(id)
            if (entity) {
              results.push(this.createResult(id, 1.0, entity))
            }
          }
          return results
        }

        // ExcludeVFS helper - exclude VFS infrastructure entities
        // VFS files/folders have vfsType set, extracted entities do NOT
        let filter: any = {}
        if (params.excludeVFS === true) {
          filter.vfsType = { exists: false }
          filter.isVFSEntity = { ne: true }
        }

        // Use metadata index when there's an actual filter (e.g. excludeVFS). The empty
        // filter returns nothing from getIdsForFilter, so the unfiltered case below uses
        // getNouns instead (it returns all nouns, including their visibility).
        if (Object.keys(filter).length > 0) {
          let filteredIds = await this.filterIdsBelted(filter)
          // Visibility hard filter — drop hidden ids BEFORE pagination.
          if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))
          const pageIds = filteredIds.slice(offset, offset + limit)

          // Batch-load entities for 10x faster cloud storage performance
          const entitiesMap = await this.batchGet(pageIds)
          for (const id of pageIds) {
            const entity = entitiesMap.get(id)
            if (entity) {
              results.push(this.createResult(id, 1.0, entity))
            }
          }
        } else {
          // No metadata filter, use direct storage query. Over-fetch by the hidden count
          // so the visibility filter below still fills the requested page (limit-exact).
          const storageResults = await this.storage.getNouns({
            pagination: { limit: limit + offset + hiddenIds.size, offset: 0 }
          })

          // Visibility hard filter on the materialized nouns (each carries `visibility`).
          const visible = hiddenIds.size > 0
            ? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id))
            : storageResults.items

          for (let i = offset; i < Math.min(offset + limit, visible.length); i++) {
            const noun = visible[i]
            if (noun) {
              const entity = await this.convertNounToEntity(noun)
              results.push(this.createResult(noun.id, 1.0, entity))
            }
          }
        }

        return results
      }

      // Metadata-first optimization: pre-resolve filter IDs before vector search.
      // This enables HNSW to search only within matching candidates instead of
      // doing expensive post-filtering. Native HNSW uses searchWithCandidates (Rust
      // bitmap), JS HNSW converts to a Set-based filter function.
      let preResolvedMetadataIds: string[] | null = null
      let preResolvedFilter: any = null
      // 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS): the matched universe as an opaque roaring
      // Buffer, forwarded straight into the vector beam walk with NO id materialization
      // when the active metadata provider can produce one (native/cor). Absent on the
      // JS path — there the materialized `candidateIds` restricts the walk instead.
      let preResolvedAllowedIds: OpaqueIdSet | undefined

      if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
        preResolvedFilter = this.buildMetadataFilter(params)
        preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)

        // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids.
        if (hiddenIds.size > 0) {
          preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id))
        }

        // Short-circuit: if metadata filter matches nothing, skip expensive vector search
        if (preResolvedMetadataIds.length === 0) {
          return []
        }

        // 8.0 #46: when the active metadata provider exposes the opaque-set producer
        // (native/cor — the JS index does not), forward the matched universe to the
        // vector beam walk as an OpaqueIdSet (zero id materialization). The opaque set
        // is the RAW filter universe (the set is opaque, so the visibility exclusion
        // cannot be AND-ed into it in TS) — hidden tiers are instead dropped by the
        // post-search visibility hard filter below, and the JS path's materialized
        // `candidateIds` stays visibility-precise. Both forms are passed; the native
        // provider consumes the opaque one, the JS index the string one.
        const mip = this.metadataIndex as unknown as MetadataIndexProvider
        if (typeof mip.getIdSetForFilter === 'function') {
          preResolvedAllowedIds = await mip.getIdSetForFilter(preResolvedFilter)
        }
      }

      // Zero-Config Hybrid Search
      // Determine search mode: auto (default) combines text + semantic for query searches
      const searchMode = params.searchMode || 'auto'
      const limit = params.limit || 10

      // Handle text-only query (user explicitly wants text search)
      if (searchMode === 'text' && params.query && params.query.trim() !== '') {
        results = await this.executeTextSearch(params.query, limit * 2)
      }
      // Handle semantic-only query (user explicitly wants vector search)
      else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) {
        results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
      }
      // Handle explicit hybrid or auto mode with query
      else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
        // Zero-config hybrid: combine text + semantic search with RRF fusion
        const [textResults, semanticResults] = await Promise.all([
          this.executeTextSearch(params.query, limit * 2),
          this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
        ])

        // Use user-specified alpha or auto-detect based on query length
        const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)

        // Tokenize query for match visibility
        const queryWords = this.metadataIndex.tokenize(params.query)

        // RRF fusion combines both result sets with match visibility
        results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords)
      }
      // Handle direct vector search (no query text) - no hybrid needed
      else if (params.vector && !params.query) {
        results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
      }
      // Handle proximity search
      else if (params.near) {
        results = await this.executeProximitySearch(params)
      }

      // Execute parallel searches for additional criteria (proximity search in addition to query)
      if (params.near && params.query) {
        const proximityResults = await this.executeProximitySearch(params)
        results.push(...proximityResults)
      }

      // Remove duplicate results from parallel searches
      if (results.length > 0) {
        const uniqueResults = new Map<string, Result<T>>()
        for (const result of results) {
          const existing = uniqueResults.get(result.id)
          if (!existing || result.score > existing.score) {
            uniqueResults.set(result.id, result)
          }
        }
        results = Array.from(uniqueResults.values())
      }

      // Visibility hard filter for vector/text/proximity candidates (the pre-resolved
      // path already excluded hidden ids; this covers searches with no metadata filter).
      // Applied BEFORE the final sort+slice so limit/offset remain exact.
      if (hiddenIds.size > 0 && results.length > 0) {
        results = results.filter((r) => !isHidden(r.id))
      }

      // Apply metadata filtering using pre-resolved IDs (metadata-first optimization).
      // When vector search was performed, HNSW already filtered by candidateIds —
      // this block handles pagination, entity loading, and metadata-only queries.
      if (preResolvedMetadataIds && preResolvedFilter) {
        const filteredIds = preResolvedMetadataIds

        if (results.length > 0) {
          // Filter results by pre-resolved metadata IDs.
          // With metadata-first HNSW, most results already match — this is a safety net
          // for text search results and proximity results that weren't pre-filtered.
          const filteredIdSet = new Set(filteredIds)
          results = results.filter((r) => filteredIdSet.has(r.id))

          // Apply early pagination for vector + metadata queries
          const limit = params.limit || 10
          const offset = params.offset || 0

          // If we have enough filtered results, sort and paginate early.
          // Rank by score (top offset+limit), then drop the offset — identical ordering
          // to a full `sort((a, b) => b.score - a.score)` + slice, but the native
          // `sort:topK` provider can compute only the page instead of the full sort.
          if (results.length >= offset + limit) {
            const k = offset + limit
            const order = rankIndicesByScore(results.map(r => r.score), k, true)
            results = reorderByIndices(results, order).slice(offset, k)

            // Batch-load entities only for the paginated results (10x faster on GCS)
            const idsToLoad = results.filter(r => !r.entity).map(r => r.id)
            if (idsToLoad.length > 0) {
              const entitiesMap = await this.batchGet(idsToLoad)
              for (const result of results) {
                if (!result.entity) {
                  const entity = entitiesMap.get(result.id)
                  if (entity) {
                    result.entity = entity
                  }
                }
              }
            }

            // Early return if no other processing needed
            if (!params.connected && !params.fusion) {
              return results
            }
          }
        } else {
          // OPTIMIZED: Apply pagination to filtered IDs BEFORE loading entities
          const limit = params.limit || 10
          const offset = params.offset || 0
          const pageIds = filteredIds.slice(offset, offset + limit)

          // Batch-load entities for current page - O(page_size) instead of O(total_results)
          // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster)
          const entitiesMap = await this.batchGet(pageIds)
          for (const id of pageIds) {
            const entity = entitiesMap.get(id)
            if (entity) {
              results.push(this.createResult(id, 1.0, entity))
            }
          }

          // Early return for metadata-only queries with pagination applied
          if (!params.query && !params.connected) {
            // Apply sorting if requested for metadata-only queries
            if (params.orderBy) {
              // Page-bounded sort (offset+limit + hidden over-fetch) so we don't
              // materialize every matching sorted id to return one page.
              const limit = params.limit || 10
              const offset = params.offset || 0
              const sortedIds = await this.metadataIndex.getSortedIdsForFilter(
                preResolvedFilter,
                params.orderBy,
                params.order || 'asc',
                offset + limit + hiddenIds.size
              )

              // Paginate sorted IDs BEFORE loading entities (production-scale!)
              const pageIds = sortedIds.slice(offset, offset + limit)

              // Batch-load entities for paginated results (10x faster on GCS)
              const sortedResults: Result<T>[] = []
              const entitiesMap = await this.batchGet(pageIds)
              for (const id of pageIds) {
                const entity = entitiesMap.get(id)
                if (entity) {
                  sortedResults.push(this.createResult(id, 1.0, entity))
                }
              }

              return sortedResults
            }

            return results
          }
        }
      }

      // Graph search component with O(1) traversal
      if (params.connected) {
        results = await this.executeGraphSearch(params, results)
      }

      // Apply fusion scoring if requested
      if (params.fusion && results.length > 0) {
        results = this.applyFusionScoring(results, params.fusion)
      }

      // OPTIMIZED: Sort first, then apply efficient pagination
      // Support custom orderBy for vector + metadata queries
      if (params.orderBy && results.length > 0) {
        // For vector + metadata queries, sort by specified field instead of score
        // Load sort field values for all results (small set, already filtered)
        const resultsWithValues = await Promise.all(results.map(async (r) => ({
          result: r,
          value: await this.metadataIndex.getFieldValueForEntity(r.id, params.orderBy!)
        })))

        // Sort by field value
        resultsWithValues.sort((a, b) => {
          // Handle null/undefined
          if (a.value == null && b.value == null) return 0
          if (a.value == null) return (params.order || 'asc') === 'asc' ? 1 : -1
          if (b.value == null) return (params.order || 'asc') === 'asc' ? -1 : 1

          // Compare values
          if (a.value === b.value) return 0
          const comparison = a.value < b.value ? -1 : 1
          return (params.order || 'asc') === 'asc' ? comparison : -comparison
        })

        results = resultsWithValues.map(({ result }) => result)
      } else {
        // Default: sort by relevance score. Rank to the page (offset+limit) via the
        // swappable sort:topK seam; the slice below drops the offset. Ordering is
        // identical to a full `sort((a, b) => b.score - a.score)`.
        const k = (params.offset || 0) + limit
        const order = rankIndicesByScore(results.map(r => r.score), k, true)
        results = reorderByIndices(results, order)
      }

      const finalOffset = params.offset || 0

      // Efficient pagination - only slice what we need (limit already defined above)
      return results.slice(finalOffset, finalOffset + limit)
    })()

    // Index-integrity guard — applied ONCE here so every find() path (metadata,
    // vector, text, proximity, graph) is covered uniformly. The indexes are
    // acceleration structures; the loaded entity is ground truth. Re-validate
    // each result against the query predicate so a stale or cross-bucket index
    // entry — e.g. an id left in a field-value posting by a delete or an
    // `update({ field: undefined })` — can never surface an entity that does not
    // actually match `type`/`subtype`/`where`/`service`/`excludeVFS`. This is
    // the live-path mirror of the historical path's per-candidate
    // `entityMatchesFind` (db.ts). On a healthy index it is a no-op; on a
    // corrupted one it drops the bad row instead of returning a phantom.
    if (result.length > 0) {
      result = result.filter((r) => {
        if (r.entity == null) return false
        try {
          return entityMatchesFind(r.entity as unknown as Entity, params as unknown as FindParams)
        } catch {
          // The JS matcher doesn't implement a where-operator the provider already
          // matched on (e.g. a future native-only operator). The guard cannot
          // DISPROVE a match the provider made, so keep the row rather than
          // hard-failing a correct provider result. (The db.ts historical path
          // keeps its throw, which legitimately reroutes to materialization.)
          return true
        }
      })
    }

    // includeVectors — opt-in vector hydration. Default (false) keeps the perf
    // contract: every result path above builds entities via the metadata-only
    // fast path, so `entity.vector` is the empty stub. When requested, fetch the
    // stored vectors for the already-paginated page in ONE batch and attach them,
    // mirroring get({ includeVectors: true }). Done once here so every find()
    // path (metadata-only, vector/text/proximity, graph) honors it uniformly.
    if (params.includeVectors && result.length > 0) {
      const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id))
      for (const r of result) {
        const noun = nounsMap.get(r.id)
        if (noun?.vector && r.entity) {
          r.entity.vector = noun.vector
        }
      }
    }

    // Record performance for auto-tuning
    const duration = Date.now() - startTime
    recordQueryPerformance(duration, result.length)

    return result
  }

  /**
   * Find similar entities using vector similarity
   *
   * @param params - Parameters specifying the target for similarity search
   * @param params.to - Entity ID, Entity object, or Vector to find similar to (required)
   * @param params.limit - Maximum results (default: 10)
   * @param params.threshold - Minimum similarity (0-1)
   * @param params.type - Filter by NounType(s)
   * @param params.where - Metadata filters
   * @returns Promise that resolves to array of Result objects with similarity scores (same structure as find())
   *
   * **Returns:**
   * Same Result structure as find() with flattened fields for convenient access
   *
   * @example
   * // Find entities similar to a specific entity by ID
   * const similarDocs = await brainy.similar({
   *   to: 'document-123',
   *   limit: 10
   * })
   *
   * // Access flattened fields
   * for (const result of similarDocs) {
   *   console.log(`Similarity: ${result.score}`)
   *   console.log(`Type: ${result.type}`)              // Flattened!
   *   console.log(`Metadata:`, result.metadata)        // Flattened!
   *   console.log(`Confidence: ${result.confidence ?? 'N/A'}`)  // Flattened!
   * }
   *
   * @example
   * // Find similar entities with type filtering
   * const similarUsers = await brainy.similar({
   *   to: 'user-456',
   *   type: NounType.Person,
   *   limit: 5,
   *   where: {
   *     active: true,
   *     department: 'engineering'
   *   }
   * })
   *
   * @example
   * // Find similar using a custom vector
   * const customVector = await brainy.embed('artificial intelligence research')
   * const similar = await brainy.similar({
   *   to: customVector,
   *   limit: 8,
   *   type: [NounType.Document, NounType.Thing]
   * })
   *
   * @example
   * // Find similar using an entity object
   * const sourceEntity = await brainy.get('research-paper-789')
   * if (sourceEntity) {
   *   const relatedPapers = await brainy.similar({
   *     to: sourceEntity,
   *     limit: 12,
   *     where: {
   *       published: true,
   *       category: 'machine-learning'
   *     }
   *   })
   * }
   *
   * @example
   * // Content recommendation system
   * async function getRecommendations(userId: string) {
   *   // Get user's recent interactions
   *   const user = await brainy.get(userId)
   *   if (!user) return []
   *
   *   // Find similar content
   *   const recommendations = await brainy.similar({
   *     to: userId,
   *     type: NounType.Document,
   *     limit: 20,
   *     where: {
   *       published: true,
   *       language: 'en'
   *     }
   *   })
   *
   *   // Filter out already seen content
   *   return recommendations.filter(rec =>
   *     !user.metadata.viewedItems?.includes(rec.id)
   *   )
   * }
   *
   * @example
   * // Duplicate detection system
   * async function findPotentialDuplicates(entityId: string) {
   *   const duplicates = await brainy.similar({
   *     to: entityId,
   *     limit: 10
   *   })
   *
   *   // High similarity might indicate duplicates
   *   const highSimilarity = duplicates.filter(d => d.score > 0.95)
   *
   *   if (highSimilarity.length > 0) {
   *     console.log('Potential duplicates found:', highSimilarity.map(d => d.id))
   *   }
   *
   *   return highSimilarity
   * }
   *
   * @example
   * // Error handling for missing entities
   * try {
   *   const similar = await brainy.similar({
   *     to: 'nonexistent-entity',
   *     limit: 5
   *   })
   * } catch (error) {
   *   if (error instanceof EntityNotFoundError) {
   *     console.log(`Source entity does not exist: ${error.id}`)
   *     // Handle missing source entity
   *   }
   * }
   */
  async similar(params: SimilarParams<T>): Promise<Result<T>[]> {
    await this.ensureInitialized()

    // Get target vector
    let targetVector: Vector

    if (typeof params.to === 'string') {
      // Need vector for similarity, so use includeVectors: true
      const entity = await this.get(params.to, { includeVectors: true })
      if (!entity) {
        throw new EntityNotFoundError(params.to)
      }
      targetVector = entity.vector
    } else if (Array.isArray(params.to)) {
      targetVector = params.to as Vector
    } else {
      // Entity object passed - check if vectors are loaded
      const entityVector = (params.to as Entity<T>).vector
      if (!entityVector || entityVector.length === 0) {
        throw new Error(
          'Entity passed to brain.similar() has no vector embeddings loaded. ' +
          'Please retrieve the entity with { includeVectors: true } or pass the entity ID instead.\n\n' +
          'Example: brain.similar({ to: entityId }) OR brain.similar({ to: await brain.get(entityId, { includeVectors: true }) })'
        )
      }
      targetVector = entityVector
    }

    // Use find with vector
    const results = await this.find({
      vector: targetVector,
      limit: params.limit,
      type: params.type,
      where: params.where,
      service: params.service,
      excludeVFS: params.excludeVFS  // Pass through VFS filtering
    })

    // A min-similarity `threshold` is applied as a post-filter on result.score
    // — the canonical way to impose a minimum score on plain semantic results
    // (top-level vector search does not honor it; see the find({ near }) guidance).
    // Previously `params.threshold` was silently dropped.
    return params.threshold != null
      ? results.filter((r) => r.score >= params.threshold!)
      : results
  }

  // ============= BATCH OPERATIONS =============

  /**
   * Add multiple entities in a single batch operation
   *
   * Uses batch embedding (embedBatch) to pre-compute all vectors before any
   * storage write, keeping model inference out of the per-item write path.
   * (On the default WASM engine, batch throughput is comparable to sequential
   * embed() calls — measured ~160 ms/text either way; a native embedding
   * provider may batch faster.) Automatically adapts batch size and
   * parallelism to the storage adapter (e.g., smaller batches for cloud
   * storage).
   *
   * @param params - Batch add parameters
   * @param params.items - Array of AddParams (same shape as brain.add())
   * @param params.parallel - Process in parallel (default: true, auto-adapts per storage)
   * @param params.chunkSize - Batch size per storage round-trip (default: auto from storage)
   * @param params.onProgress - Callback: (completed, total) => void
   * @param params.continueOnError - If true, skip failed items instead of aborting
   * @returns BatchResult with successful (string[] of IDs), failed, total, duration
   *
   * @example
   * ```typescript
   * const result = await brain.addMany({
   *   items: [
   *     { data: 'First entity', type: NounType.Document, metadata: { priority: 1 } },
   *     { data: 'Second entity', type: NounType.Concept }
   *   ],
   *   onProgress: (done, total) => console.log(`${done}/${total}`)
   * })
   * console.log(`Added ${result.successful.length}, failed ${result.failed.length}`)
   * ```
   */
  async addMany(params: AddManyParams<T>): Promise<BatchResult<string>> {
    this.assertWritable('addMany')
    await this.ensureInitialized()

    // Pre-validate every item against per-type + brain-wide subtype rules BEFORE
    // any storage write — atomic-fail semantics: a missing-subtype anywhere in
    // the batch fails the whole call, no partial writes. (7.30.0)
    for (let i = 0; i < params.items.length; i++) {
      const item = params.items[i]
      try {
        this.enforceSubtypeOnAdd('add', item.type, item.subtype, item.metadata)
      } catch (err) {
        const msg = err instanceof Error ? err.message : String(err)
        throw new Error(`addMany(): item[${i}] failed subtype enforcement: ${msg}`)
      }
    }

    // Get optimal batch configuration from storage adapter
    // This automatically adapts to storage characteristics:
    // - GCS: 50 batch size, 100ms delay, sequential
    // - S3/R2: 100 batch size, 50ms delay, parallel
    // - Memory: 1000 batch size, 0ms delay, parallel
    const storageConfig = this.storage.getBatchConfig()

    // Use storage preferences (allow explicit user override)
    const batchSize = params.chunkSize ?? storageConfig.maxBatchSize
    const parallel = params.parallel ?? storageConfig.supportsParallelWrites
    const delayMs = storageConfig.batchDelayMs

    const result: BatchResult<string> = {
      successful: [],
      failed: [],
      total: params.items.length,
      duration: 0
    }

    const startTime = Date.now()
    let lastBatchTime = Date.now()

    // OPTIMIZATION: Pre-compute vectors using batch embedding
    // Items that already have vectors are skipped
    // This changes N individual WASM calls → 1 batched WASM call (5-10x faster)
    const itemsNeedingEmbedding: { index: number; text: string }[] = []

    for (let i = 0; i < params.items.length; i++) {
      const item = params.items[i]
      if (!item.vector && item.data !== undefined && item.data !== null) {
        // Convert data to string for embedding
        const text = typeof item.data === 'string'
          ? item.data
          : JSON.stringify(item.data)
        itemsNeedingEmbedding.push({ index: i, text })
      }
    }

    // Batch embed all texts that need vectors
    if (itemsNeedingEmbedding.length > 0) {
      const texts = itemsNeedingEmbedding.map(item => item.text)
      const vectors = await this.embedBatch(texts)

      // Attach pre-computed vectors to items
      for (let i = 0; i < itemsNeedingEmbedding.length; i++) {
        const { index } = itemsNeedingEmbedding[i]
        // Mutate the item to include the pre-computed vector
        // This way add() will skip embedding (vector already provided)
        params.items[index].vector = vectors[i]
      }
    }

    // OPTIMIZATION: Defer HNSW persistence during batch insert.
    // Without this, each add() triggers ~16-20 neighbor saveVectorIndexData calls
    // (each a read→gzip→atomic-write cycle). For 450 items that's ~8,100
    // individual storage writes. Deferred mode collects dirty node IDs and
    // flushes once at the end — deduplicating repeated neighbor updates.
    const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks
    const prevPersistMode = typeof index.getPersistMode === 'function'
      ? index.getPersistMode()
      : null
    const canDefer = prevPersistMode === 'immediate'
      && typeof index.setPersistMode === 'function'
      && typeof index.flush === 'function'

    if (canDefer) {
      // canDefer verified setPersistMode is a function just above.
      index.setPersistMode!('deferred')
    }

    try {
      // Process in batches
      for (let i = 0; i < params.items.length; i += batchSize) {
        const chunk = params.items.slice(i, i + batchSize)

        const promises = chunk.map(async (item) => {
          try {
            // ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every
            // item, but let a per-item flag take precedence so callers can override
            // individual rows. The two flags stay independent: per-item upsert handling
            // (create-or-merge) and per-item ifAbsent handling (create-or-skip) both
            // fire inside the delegated add() call, which also enforces their mutual
            // exclusion when a single resolved item carries both.
            const resolvedItem =
              (params.ifAbsent && item.ifAbsent === undefined) ||
              (params.upsert && item.upsert === undefined)
                ? {
                    ...item,
                    ...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }),
                    ...(params.upsert && item.upsert === undefined && { upsert: true })
                  }
                : item
            const id = await this.add(resolvedItem)
            result.successful.push(id)
          } catch (error) {
            result.failed.push({
              item,
              error: (error as Error).message
            })
            if (!params.continueOnError) {
              throw error
            }
          }
        })

        // Parallel vs Sequential based on storage preference
        if (parallel) {
          await Promise.allSettled(promises)
        } else {
          // Sequential processing for rate-limited storage
          for (const promise of promises) {
            await promise
          }
        }

        // Progress callback
        if (params.onProgress) {
          params.onProgress(
            result.successful.length + result.failed.length,
            result.total
          )
        }

        // Adaptive delay between batches
        if (i + batchSize < params.items.length && delayMs > 0) {
          const batchDuration = Date.now() - lastBatchTime

          // If batch was too fast, add delay to respect rate limits
          if (batchDuration < delayMs) {
            await new Promise(resolve =>
              setTimeout(resolve, delayMs - batchDuration)
            )
          }

          lastBatchTime = Date.now()
        }
      }
    } finally {
      // Restore persist mode and flush all dirty nodes in one pass.
      // canDefer implies prevPersistMode === 'immediate', so the null re-check
      // never fires at runtime — it narrows the type for the restore call.
      if (canDefer && prevPersistMode !== null) {
        await index.flush()
        // canDefer verified setPersistMode is a function before the try block.
        index.setPersistMode!(prevPersistMode)
      }
    }

    result.duration = Date.now() - startTime
    return result
  }

  /**
   * Remove multiple entities
   */
  async removeMany(params: RemoveManyParams): Promise<BatchResult<string>> {
    this.assertWritable('removeMany')
    await this.ensureInitialized()

    // Loud selector validation: a call with no usable selector used to
    // resolve successfully having deleted NOTHING (total: 0) — the classic
    // silent no-op being a bare array passed positionally
    // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller
    // believes the delete happened; every count derived afterwards is "wrong"
    // while the engine was never even asked. Refuse instead.
    if (Array.isArray(params)) {
      throw new Error(
        `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })`
      )
    }
    if (!params || (!params.ids && !params.type && !params.where)) {
      throw new Error(
        `removeMany() requires a selector: { ids } and/or { type, where }. ` +
          `An empty selector would silently delete nothing — refusing.`
      )
    }
    // An empty `where: {}` carries zero predicates. find() serves it as
    // MATCH-ALL (the served-or-refused law), which on this destructive path
    // would silently become "delete up to `limit` arbitrary rows". A bulk
    // delete of everything must be asked for explicitly (type selector, real
    // predicates, or ids) — refuse the ambiguous shape loudly.
    if (params.where && !params.ids && !params.type && !whereConstrains(params.where)) {
      throw new Error(
        `removeMany() received where: {} — an empty filter matches EVERYTHING, ` +
          `and a match-all bulk delete must be explicit. Pass real predicates, ` +
          `a { type }, or { ids }; to clear the store use clear().`
      )
    }
    if (params.ids && params.ids.length === 0) {
      throw new Error(
        `removeMany() received ids: [] — an empty id list deletes nothing. ` +
          `Pass the ids to delete, or omit ids and select by { type, where }.`
      )
    }

    // Determine what to delete
    let idsToDelete: string[] = []

    if (params.ids) {
      // Id normalization (8.0): resolve each supplied id to the canonical UUID
      // add() stored, so callers may delete by natural key. The find()-derived
      // path below already yields canonical ids.
      idsToDelete = params.ids.map((id) => resolveEntityId(id))
    } else if (params.type || params.where) {
      // Find entities to delete
      const entities = await this.find({
        type: params.type,
        where: params.where,
        limit: params.limit || 1000
      })
      idsToDelete = entities.map((e) => e.id)
    }

    const result: BatchResult<string> = {
      successful: [],
      failed: [],
      total: idsToDelete.length,
      duration: 0
    }

    const startTime = Date.now()

    // Batch deletes into chunks for 10x faster performance with proper error handling.
    // Single transaction per chunk = atomic within chunk, graceful failure across chunks.
    // Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's
    // maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it.
    const storageConfig = this.storage.getBatchConfig()
    const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize

    for (let i = 0; i < idsToDelete.length; i += chunkSize) {
      const chunk = idsToDelete.slice(i, i + chunkSize)

      // Track IDs queued during builder phase separately from confirmed deletions.
      // result.successful must only be updated AFTER the transaction commits — pushing
      // inside the builder runs before transaction.execute(), so a rollback would leave
      // successfully-queued IDs incorrectly listed as deleted.
      const chunkQueued: string[] = []
      const chunkBuilderFailed: Array<{ item: string; error: string }> = []

      // Model-B pre-pass: collect the chunk's touched-id set (entities + their
      // cascade-deleted relationships) so the generation captures before-images
      // for all of them. Tolerant by design — the builder below still does the
      // authoritative per-id load + error handling, and an over-included id whose
      // delete is skipped simply gets a before-image equal to its live state (a
      // harmless no-op history entry).
      const touchedNouns: string[] = [...chunk]
      const touchedVerbs: string[] = []
      for (const id of chunk) {
        try {
          const srcVerbs = await this.storage.getVerbsBySource(id)
          const tgtVerbs = await this.storage.getVerbsByTarget(id)
          for (const v of [...srcVerbs, ...tgtVerbs]) touchedVerbs.push(v.id)
        } catch {
          // Ignore — the builder's per-id try/catch is authoritative.
        }
      }

      try {
        // Change feed: the BUILDER populates this (per id actually staged), so
        // an id whose builder failed never emits — the array is read only
        // after the chunk's commit succeeds. Verb events dedupe within the
        // chunk (two related chunk-members see the same cascade verb twice).
        const chunkEvents: PendingChangeEvent[] | undefined = this._changeFeed
          .hasListeners
          ? []
          : undefined
        const seenVerbEvents = new Set<string>()

        // Process chunk as ONE atomic Model-B generation (entities + cascade verbs).
        await this.persistSingleOp({ nouns: touchedNouns, verbs: touchedVerbs }, async (tx) => {
          for (const id of chunk) {
            try {
              // Load entity data
              const metadata = await this.storage.getNounMetadata(id)
              const noun = await this.storage.getNoun(id)
              const verbs = await this.storage.getVerbsBySource(id)
              const targetVerbs = await this.storage.getVerbsByTarget(id)
              const allVerbs = [...verbs, ...targetVerbs]

              // Add delete operations to transaction
              if (noun) {
                tx.addOperation(
                  new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)
                )
              }

              if (metadata) {
                tx.addOperation(
                  new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration)
                )
              }

              // Pre-read metadata rides along: the count decrement must not
              // depend on re-reading the record being removed (see remove()).
              tx.addOperation(
                new DeleteNounMetadataOperation(this.storage, id, metadata)
              )

              for (const verb of allVerbs) {
                const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
                tx.addOperation(
                  new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration)
                )
                tx.addOperation(
                  new DeleteVerbMetadataOperation(this.storage, verb.id)
                )
              }

              if (chunkEvents) {
                chunkEvents.push({
                  kind: 'entity',
                  op: 'remove',
                  id,
                  ...(metadata && {
                    entity: this.entityViewFromRawRecord(
                      id,
                      metadata as Record<string, unknown>
                    )
                  })
                })
                for (const verb of allVerbs) {
                  if (seenVerbEvents.has(verb.id)) continue
                  seenVerbEvents.add(verb.id)
                  chunkEvents.push({
                    kind: 'relation',
                    op: 'unrelate',
                    id: verb.id,
                    relation: {
                      id: verb.id,
                      from: verb.sourceId,
                      to: verb.targetId,
                      type: String(verb.verb)
                    }
                  })
                }
              }

              chunkQueued.push(id)
            } catch (error) {
              chunkBuilderFailed.push({
                item: id,
                error: (error as Error).message
              })
              if (!params.continueOnError) {
                throw error
              }
            }
          }
        }, undefined, chunkEvents)

        // Transaction committed — queued IDs were actually deleted
        result.successful.push(...chunkQueued)
        result.failed.push(...chunkBuilderFailed)
      } catch (error) {
        // Transaction failed/rolled back — queued IDs were NOT deleted
        result.failed.push(...chunkBuilderFailed)
        for (const id of chunkQueued) {
          result.failed.push({
            item: id,
            error: (error as Error).message
          })
        }

        // Stop processing if continueOnError is false
        if (!params.continueOnError) {
          break
        }
      }

      if (params.onProgress) {
        params.onProgress(
          result.successful.length + result.failed.length,
          result.total
        )
      }
    }

    result.duration = Date.now() - startTime
    return result
  }

  /**
   * Update multiple entities with batch processing
   */
  async updateMany(params: UpdateManyParams<T>): Promise<BatchResult<string>> {
    await this.ensureInitialized()

    const result: BatchResult<string> = {
      successful: [],
      failed: [],
      total: params.items.length,
      duration: 0
    }

    const startTime = Date.now()
    const chunkSize = params.chunkSize || 100

    // Process in chunks
    for (let i = 0; i < params.items.length; i += chunkSize) {
      const chunk = params.items.slice(i, i + chunkSize)

      const promises = chunk.map(async (item, chunkIndex) => {
        try {
          await this.update(item)
          result.successful.push(item.id)
        } catch (error) {
          result.failed.push({
            item,
            error: (error as Error).message
          })
          if (!params.continueOnError) {
            throw error
          }
        }
      })

      if (params.parallel !== false) {
        await Promise.allSettled(promises)
      } else {
        for (const promise of promises) {
          await promise
        }
      }

      // Report progress
      if (params.onProgress) {
        params.onProgress(
          result.successful.length + result.failed.length,
          result.total
        )
      }
    }

    result.duration = Date.now() - startTime
    return result
  }

  /**
   * Create multiple relationships in a single batch operation
   *
   * Automatically adapts batch size and parallelism to the storage adapter.
   * Duplicate relationships are detected and deduplicated per relate().
   *
   * @param params - Batch relate parameters
   * @param params.items - Array of RelateParams (same shape as brain.relate())
   * @param params.parallel - Process in parallel (default: true, auto-adapts per storage)
   * @param params.chunkSize - Batch size per storage round-trip (default: auto from storage)
   * @param params.onProgress - Callback: (completed, total) => void
   * @param params.continueOnError - If true, skip failed items instead of aborting
   * @returns Array of relationship IDs (string[])
   *
   * @example
   * ```typescript
   * const ids = await brain.relateMany({
   *   items: [
   *     { from: id1, to: id2, type: VerbType.RelatedTo },
   *     { from: id1, to: id3, type: VerbType.Contains, data: 'section content' }
   *   ]
   * })
   * ```
   */
  async relateMany(params: RelateManyParams<T>): Promise<string[]> {
    this.assertWritable('relateMany')
    await this.ensureInitialized()

    // Pre-validate every item against per-type + brain-wide subtype rules BEFORE
    // any storage write — atomic-fail semantics: a missing-subtype anywhere in
    // the batch fails the whole call, no partial writes. (7.30.0)
    for (let i = 0; i < params.items.length; i++) {
      const item = params.items[i]
      try {
        this.enforceSubtypeOnRelate('relate', item.type, item.subtype, item.metadata)
      } catch (err) {
        const msg = err instanceof Error ? err.message : String(err)
        throw new Error(`relateMany(): item[${i}] failed subtype enforcement: ${msg}`)
      }
    }

    // Get optimal batch configuration from storage adapter
    // Automatically adapts to storage characteristics
    const storageConfig = this.storage.getBatchConfig()

    // Use storage preferences (allow explicit user override)
    const batchSize = params.chunkSize ?? storageConfig.maxBatchSize
    const parallel = params.parallel ?? storageConfig.supportsParallelWrites
    const delayMs = storageConfig.batchDelayMs

    const result: BatchResult<string> = {
      successful: [],
      failed: [],
      total: params.items.length,
      duration: 0
    }

    const startTime = Date.now()
    let lastBatchTime = Date.now()

    for (let i = 0; i < params.items.length; i += batchSize) {
      const chunk = params.items.slice(i, i + batchSize)

      if (parallel) {
        // Parallel processing
        const promises = chunk.map(async (item) => {
          try {
            const relationId = await this.relate(item)
            result.successful.push(relationId)
          } catch (error: any) {
            result.failed.push({
              item,
              error: error.message || 'Unknown error'
            })
            if (!params.continueOnError) {
              throw error
            }
          }
        })
        await Promise.allSettled(promises)
      } else {
        // Sequential processing
        for (const item of chunk) {
          try {
            const relationId = await this.relate(item)
            result.successful.push(relationId)
          } catch (error: any) {
            result.failed.push({
              item,
              error: error.message || 'Unknown error'
            })
            if (!params.continueOnError) {
              throw error
            }
          }
        }
      }

      // Progress callback
      if (params.onProgress) {
        params.onProgress(
          result.successful.length + result.failed.length,
          result.total
        )
      }

      // Adaptive delay
      if (i + batchSize < params.items.length && delayMs > 0) {
        const batchDuration = Date.now() - lastBatchTime
        if (batchDuration < delayMs) {
          await new Promise(resolve =>
            setTimeout(resolve, delayMs - batchDuration)
          )
        }
        lastBatchTime = Date.now()
      }
    }

    result.duration = Date.now() - startTime
    return result.successful
  }

  /**
   * @description Clear ALL data from the database — entities, relationships,
   * blobs, and every derived index. The instance remains fully usable
   * afterwards: the vector, metadata, and graph indexes are re-resolved
   * exactly as on `init()`, and when the VFS was active a fresh VFS root
   * entity is re-created (so a thresholdless vector search against a cleared
   * store can still surface that one root entity).
   * @example
   * await brain.clear()
   */
  async clear(): Promise<void> {
    await this.ensureInitialized()
    // A clear mutates durable state without going through a commit path, so
    // it must set the dirty witness itself — otherwise a `clear()` followed by
    // `flush()` would find the brain "clean" and skip the entity-tree stamp,
    // leaving a stamp that describes the population this call just removed.
    this._dirtySinceLastFlush = true

    // Clear storage
    await this.storage.clear()

    // Invalidate GraphAdjacencyIndex to prevent stale in-memory data
    // The index has LSMTree data and verbIdSet pointing to deleted entities.
    // Without this, relate()'s duplicate check uses stale data, potentially
    // allowing duplicate relationships or missing valid duplicates.
    if (typeof this.storage.invalidateGraphIndex === 'function') {
      this.storage.invalidateGraphIndex()
    }

    // Reset index
    if ('clear' in this.index && typeof this.index.clear === 'function') {
      await this.index.clear()
    } else {
      // Recreate index using plugin factory when available
      this.index = this.createIndex()
    }

    // Recreate metadata index to clear cached data — mirror init()'s
    // resolution: provider factory first, then the JS implementation (with a
    // plugin-provided native entityIdMapper when registered).
    const clearMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
    if (clearMetadataFactory) {
      this.metadataIndex = clearMetadataFactory(this.storage)
    } else {
      const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper')
      this.metadataIndex = new MetadataIndexManager(this.storage, {}, {
        entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined,
      })
    }
    await this.metadataIndex.init()

    // Re-resolve the graph index the same way init() does (provider factory,
    // else the storage layer's lazy singleton) and re-wire the shared
    // UUID ↔ int resolver. Without this, every graph-touching call after
    // clear() — relate(), getNeighbors(), stats() — would hit an undefined
    // index.
    const clearGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
    if (clearGraphFactory) {
      this.graphIndex = clearGraphFactory(this.storage)
      this.storage.setGraphIndex(this.graphIndex)
    } else {
      this.graphIndex = await this.storage.getGraphIndex()
    }
    this.wireGraphIdResolver()

    // Reset dimensions
    this.dimensions = undefined

    // Clear any cached sub-APIs
    this._nlp = undefined
    this._tripleIntelligence = undefined

    // Re-initialize the content-addressed blob store after storage.clear()
    // (clear() drops the BlobStorage instance). The VFS stores all file
    // content through it, so this must happen BEFORE VFS reinitialization.
    if (typeof this.storage.initializeBlobStorage === 'function') {
      await this.storage.initializeBlobStorage()
    }

    // Reset VFS state - root entity was deleted by storage.clear()
    // Bug: VFS instance remained in memory pointing to deleted root entity
    if (this._vfs) {
      // Clear PathResolver caches (including UnifiedCache VFS entries).
      // Boundary: peeks at VirtualFileSystem's private `pathResolver` member —
      // VFS exposes no public cache-invalidation surface, and the resolver is
      // about to be discarded with the VFS instance recreated below.
      const vfsInternals = this._vfs as unknown as {
        pathResolver?: { invalidateAllCaches?: () => void }
      }
      if (vfsInternals.pathResolver?.invalidateAllCaches) {
        vfsInternals.pathResolver.invalidateAllCaches()
      }
      // Recreate and reinitialize VFS so it's ready for use
      this._vfs = new VirtualFileSystem(this)
      await this._vfs.init()
      // _vfsInitialized remains true since we just initialized
    } else {
      // VFS was never used, reset flag for clean state
      this._vfsInitialized = false
    }

    // Change feed: clear() wipes the store wholesale (raw, not per-record) —
    // one store-level event tells subscribers everything they held is gone.
    this._changeFeed.emit([
      { kind: 'store', op: 'clear', timestamp: Date.now() }
    ])
  }

  // ─── Migration API ───────────────────────────────────────────────

  /**
   * Run pending data migrations, or preview what would change.
   *
   * @param options - Pass `{ dryRun: true }` to preview without writing;
   *   pass `{ backupTo: path }` to persist a pre-migration snapshot.
   * @returns Migration result (or preview if dryRun)
   *
   * Safety: with `backupTo`, a hard-link snapshot of the current generation
   * is persisted BEFORE any transform runs (the Db API's `persist()`), and
   * `brain.restore(path, { confirm: true })` brings it back wholesale.
   * Transforms are also idempotent (they return `null` when already
   * applied), so re-running migrations is safe.
   *
   * @example
   * ```typescript
   * // Preview what would change
   * const preview = await brain.migrate({ dryRun: true })
   * console.log(preview.affectedEntities)
   *
   * // Apply migrations with a restore point
   * const result = await brain.migrate({ backupTo: '/backups/pre-migration' })
   * console.log(result.entitiesModified, result.backupPath)
   * ```
   */
  async migrate(options?: MigrateOptions): Promise<MigrationResult | MigrationPreview> {
    await this.ensureInitialized()
    const runner = this._pendingMigrationRunner || new MigrationRunner(this.storage)

    if (options?.dryRun) {
      return runner.preview()
    }

    return this.migrateInternal(runner, options)
  }

  /**
   * Check for pending migrations during init().
   * Runs inline for small datasets if autoMigrate is enabled,
   * otherwise logs a warning.
   */
  private async checkMigrations(): Promise<void> {
    const runner = new MigrationRunner(this.storage)

    if (!(await runner.hasPendingMigrations())) {
      return
    }

    const count = await runner.pendingCount()

    if (this.config.autoMigrate) {
      // Quick entity count check to decide inline vs deferred
      const probe = await this.storage.getNouns({ pagination: { limit: 1 } })
      const totalEstimate = probe.totalCount ?? (probe.hasMore ? 10001 : probe.items.length)

      if (totalEstimate < 10000) {
        // Small dataset — migrate inline during init
        await this.migrateInternal(runner)
      } else {
        // Large dataset — defer to explicit brain.migrate() call
        this._pendingMigrationRunner = runner
        if (!this.config.silent) {
          console.log(`[brainy] ${count} pending migration(s) detected. Call brain.migrate() to apply (dataset too large for inline migration).`)
        }
      }
    } else {
      if (!this.config.silent) {
        console.log(`[brainy] ${count} pending migration(s) available. Set autoMigrate: true or call brain.migrate() to apply.`)
      }
    }
  }

  /**
   * Internal: run pending migrations and rebuild the metadata index when
   * anything changed. With `backupTo`, a hard-link snapshot of the current
   * generation is persisted before any transform runs — the Db-API
   * replacement for the pre-8.0 automatic backup branches. Transforms are
   * idempotent (return `null` when already applied), so re-runs are safe.
   */
  private async migrateInternal(runner: MigrationRunner, options?: MigrateOptions): Promise<MigrationResult> {
    let backupPath: string | null = null
    if (options?.backupTo) {
      const pin = this.now()
      try {
        await pin.persist(options.backupTo)
        backupPath = options.backupTo
      } finally {
        await pin.release()
      }
    }

    const runResult = await runner.run({ onProgress: options?.onProgress, maxErrors: options?.maxErrors })

    // Rebuild MetadataIndex if any entities were modified
    if (runResult.entitiesModified > 0) {
      await this.metadataIndex.rebuild()
    }

    // Clear deferred runner
    this._pendingMigrationRunner = undefined

    return { backupPath, ...runResult }
  }

  // ============= 8.0 DB API — GENERATIONAL MVCC =============
  //
  // Datomic-style immutable database values over the generational record
  // layer (src/db/). `now()` pins the current generation in O(1);
  // `transact()` commits a declarative batch atomically as exactly one
  // generation; `asOf()` opens past generations, timestamps, or persisted
  // snapshots; `compactHistory()` reclaims unpinned history. The `Db` value
  // type lives in src/db/db.ts; this region is the brain-side host:
  // pin/release plumbing (storage + versioned index providers), record
  // materialization, the transact planner, and snapshot/restore.

  /**
   * @description The store's current generation — a monotonic u64 watermark
   * bumped once per committed `transact()` batch and once per
   * single-operation write batch (`add`/`update`/`remove`/`relate`/…).
   * Persisted in `_system/generation.json`; never reused, including across
   * restarts and `restore()`.
   *
   * @returns The current generation number.
   * @throws Error when called before `init()` completes.
   * @example
   * const before = brain.generation()
   * await brain.transact([{ op: 'add', type: NounType.Document, data: 'x', subtype: 'note' }])
   * brain.generation() // before + 1
   */
  generation(): number {
    this.assertGenerationStoreReady('generation')
    return this.generationStore.generation()
  }

  /**
   * @description The data-layer + derived-index format this brain instance runs
   * as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the
   * compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH}
   * constants (the single source of truth shared with the native provider, in
   * `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any
   * on-disk drift and IS running the current format, so this reports what it
   * runs as, not necessarily the (possibly older) marker it opened.
   *
   * This is a pure in-memory constant read — no `await`, no storage I/O — by
   * design: a native metadata/index provider calls it synchronously at its own
   * `init()` (which runs during this brain's provider-construction phase, after
   * the marker has been loaded) to confirm "running data-format X, index epoch
   * N" before binding its native readers.
   *
   * The on-disk marker (`_system/brain-format.json`) is reconciled separately:
   * on open Brainy compares the on-disk `indexEpoch` against
   * {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the
   * derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`).
   *
   * @returns `{ dataFormat, indexEpoch }` for the running build.
   * @example
   * const { dataFormat, indexEpoch } = brain.formatInfo()
   * // dataFormat === '8.0', indexEpoch === 1
   */
  formatInfo(): BrainFormat {
    return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
  }

  /**
   * @description Stamp `_system/brain-format.json` with this build's
   * `{ dataFormat, indexEpoch }` ({@link CURRENT_DATA_FORMAT} /
   * {@link EXPECTED_INDEX_EPOCH}). A native provider calls this once its
   * background index migration (the no-freeze path, where brainy DEFERRED the
   * rebuild because the provider's `isMigrating()` was true) has
   * verified-and-swapped all derived indexes — brainy authors `dataFormat`, the
   * provider never does. The marker is the single switch that declares the
   * on-disk derived indexes current for this build's epoch, so it is advanced
   * only after the indexes it certifies are in place. Unconditional and
   * idempotent (writes the same two compiled constants every call); a no-op for
   * read-only brains, which never author markers.
   * @returns Resolves once the marker is durable on disk.
   * @example
   * // cor, after its build-new→verify→swap completes:
   * await brain.stampBrainFormat()
   */
  async stampBrainFormat(): Promise<void> {
    if (this.isReadOnly) return
    await writeBrainFormat(this.storage)
    this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
    this._indexEpochStale = false
    // The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if
    // none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker).
    await this.removeMigrationBackupSafe()
  }

  /**
   * @description Open a sequential scan over the generation FACT LOG — the
   * append-only record of every committed generation as an AFTER-IMAGE fact
   * (what each touched entity/relationship became, or a body-less tombstone
   * for a removal). The scan is the streaming substrate for index heals and
   * incremental replays: one sequential read in commit order replaces a
   * per-entity directory walk. The handle carries heal-narration telemetry
   * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered
   * batches each stamped with their generation range, byte size, and segment;
   * a `summary()` cross-check after iteration). A detected gap or damaged
   * segment ABORTS the scan loudly — never a silent skip.
   *
   * Returns `null` when this store hosts no fact log: the storage adapter
   * lacks the binary append primitives, or the brain predates the fact log
   * (its history began before dual-write — facts exist only from the first
   * write after upgrade; callers fall back to the canonical enumeration walk).
   *
   * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive
   *   both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps
   *   facts per yielded batch (default 256).
   * @returns The scan handle, or `null` when no fact log exists.
   * @example
   * const scan = brain.scanFacts({ fromGeneration: 1 })
   * if (scan) {
   *   for await (const batch of scan.batches()) {
   *     for (const fact of batch.facts) {
   *       // fact.ops: [{ kind, id, record | null (tombstone) }, ...]
   *     }
   *   }
   * }
   */
  scanFacts(options?: {
    fromGeneration?: number
    toGeneration?: number
    kinds?: Array<'noun' | 'verb'>
    batchSize?: number
  }): FactScanHandle | null {
    const factLog = this.generationStore?.getFactLog()
    return factLog ? factLog.scanFacts(options) : null
  }

  /**
   * @description The immutable, sealed fact-log segment files covering
   * `fromGeneration` — the zero-copy handoff for consumers that map segment
   * files directly instead of streaming {@link scanFacts} batches. The
   * append-mutable TAIL segment is deliberately excluded (read it via
   * `scanFacts`). Paths are storage-root-relative. Empty when no fact log
   * exists or nothing is sealed yet.
   */
  factSegmentPaths(options?: { fromGeneration?: number }): string[] {
    return this.generationStore?.getFactLog()?.segmentPaths(options) ?? []
  }

  /**
   * @description This brain's storage authority as read at open: `'tree'`
   * (the canonical record tree is authoritative; the generation log is a
   * complete dual-written journal — the default) or `'log'` (the log is
   * authoritative; single-op acks are durable-at-ack). See
   * {@link adoptLogAuthority} for the guarded flip.
   */
  logAuthority(): LogAuthorityRecord {
    return { ...this._logAuthority }
  }

  /**
   * @description Run the log-completeness VERIFICATION ORACLE (read-only):
   * replay the generation log and diff the resulting per-id state against
   * the canonical tree. Green = the log exactly reproduces canonical truth.
   * Red NAMES every divergence class — `pre-log-record` rows (canonical
   * history the log never saw) need a baseline backfill before this brain
   * can ever flip. Safe at any time; walks are paged and memory-bounded
   * (digests, never bodies).
   */
  async verifyLogAuthority(): Promise<OracleReport> {
    return this.runOracle()
  }

  /**
   * The oracle run behind {@link Brainy.verifyLogAuthority}; the adoption
   * backfill calls it with `listAll` so one scan yields the ENTIRE curable
   * mismatch set instead of the wire-capped first 200.
   */
  private async runOracle(options?: { listAll?: boolean }): Promise<OracleReport> {
    await this.ensureInitialized()
    return runLogCompletenessOracle({
      storage: this.storage as unknown as LogAuthorityStorage,
      scanFacts: () => this.scanFacts(),
      ...(options?.listAll ? { mismatchListCap: Number.POSITIVE_INFINITY } : {}),
      // Both sides normalize to ENTITY TRUTH before digesting: canonical
      // wrappers denormalize HNSW residue (connections/level) the log never
      // carries — digesting it would fake state-differs on any nonzero-level
      // node (the residue has its own rebuild path; it is not entity state).
      canonicalNounDigest: async (id: string) => {
        const raw = await this.storage.readNounRaw(id)
        if (raw.metadata === null && raw.vector === null) return null
        return recordDigest(nounEntityTruth({ metadata: raw.metadata, vector: raw.vector }))
      },
      factRecordDigest: (record: unknown) =>
        recordDigest(nounEntityTruth(record as { metadata: unknown; vector: unknown }))
    })
  }

  /**
   * @description THE GUARDED FLIP: run the oracle; on GREEN, persist the
   * authority switch and enable durable-at-ack immediately (the rest of
   * log-authoritative behavior engages at the next open — the switch is
   * checked-at-open by law). On RED the flip REFUSES, naming the first
   * divergence and the cure. One-directional unless an operator reverts
   * the stored artifact explicitly.
   * @returns The oracle report (green) — callers surface it as the flip receipt.
   * @throws When the oracle is red; nothing is written.
   */
  async adoptLogAuthority(): Promise<OracleReport> {
    await this.ensureInitialized()
    this.assertWritable('adoptLogAuthority')
    // Fold-checkpoint chain, phase 1: a FRESH brain (no committed
    // generations) arms the chain now so the backfill's re-commits below
    // feed the canonical-sync accumulator — its first stamp is then total.
    // A non-fresh flip skips (the store refuses the arm); its chain starts
    // at the first recovery fold instead. Disarmed on any failure below.
    this.generationStore.beginFoldCheckpointBootstrap()
    try {
      return await this.adoptLogAuthorityInner()
    } catch (err) {
      this.generationStore.abandonFoldCheckpointBootstrap()
      throw err
    }
  }

  /** The adoption body — see {@link Brainy.adoptLogAuthority} (which owns the
   *  fold-checkpoint bootstrap arm/disarm around it). */
  private async adoptLogAuthorityInner(): Promise<OracleReport> {
    let report = await this.runOracle({ listAll: true })

    // BASELINE BACKFILL: curable divergences are rows whose CANONICAL truth
    // simply never reached the log — pre-log records (e.g. the generation-0
    // VFS root, or a brain older than its log) and witness drift from
    // maintenance that rewrote canonical outside a generation. The cure is
    // an identity re-commit: any generational touch of the row makes the
    // commit fact capture the CURRENT canonical bytes (the fact reads
    // canonical back after execute), so the log converges on witness truth.
    // Log-AHEAD divergences (log-live-canonical-absent /
    // log-tombstone-canonical-present) are NOT curable by backfill — the
    // log claims things the witness denies — and refuse loudly below.
    //
    // RUNS TO COMPLETION. Each pass sees the ENTIRE curable set (the oracle
    // is run uncapped here) and cures all of it, so a pre-log baseline of
    // any size adopts in ONE call — the only stop is the no-progress guard.
    // A production brain with a 12.7k-row baseline once advanced exactly
    // 800 rows per call (a five-pass ceiling × the 200-row wire cap) and sat
    // tree-authoritative for hours; the bound was sized for drift, never
    // for a baseline. Pace rides the write path now: one full-brain scan
    // per pass amortizes over thousands of cures, not two hundred.
    let passes = 0
    for (;;) {
      if (report.verdict !== 'red') break
      passes++
      const curable = report.mismatches.filter(
        (m) => m.reason === 'pre-log-record' || m.reason === 'state-differs'
      )
      const incurable = report.mismatches.filter(
        (m) => m.reason !== 'pre-log-record' && m.reason !== 'state-differs'
      )
      if (incurable.length > 0) {
        throw new Error(
          `adoptLogAuthority(): the log claims state the canonical witness denies ` +
            `(${incurable.length} divergence(s); first: ${incurable[0].reason} on ` +
            `${incurable[0].id}) — backfill cannot cure a log-ahead divergence. ` +
            `Investigate before flipping; the witness remains authoritative.`
        )
      }
      if (curable.length === 0) break
      prodLog.info(
        `[Brainy] adoptLogAuthority: baseline backfill pass ${passes} — re-committing ` +
          `${curable.length} row(s) whose canonical truth never reached the log`
      )
      // Progress narration for a live operator: a large baseline is minutes
      // of visible motion, never a silent wait.
      const narrateEvery = curable.length >= 2000 ? 1000 : curable.length >= 400 ? 200 : 0
      let cured = 0
      for (const m of curable) {
        const raw = await this.storage.readNounRaw(m.id)
        if (raw.metadata === null && raw.vector === null) continue // vanished since the scan
        cured++
        if (narrateEvery > 0 && cured % narrateEvery === 0) {
          prodLog.info(
            `[Brainy] adoptLogAuthority: backfill pass ${passes} — ${cured}/${curable.length} rows re-committed`
          )
        }
        // LAW-SHAPE RE-COMMIT: rewrite canonical as EXACTLY the wrapper the
        // log's reconstruction produces (the hydration law: denormalized
        // enumeration fields derived from the metadata leg + the embedding
        // floats). This is what makes the backfill actually CURE
        // state-differs drift: rows written before the hydration law carry
        // denormalized copies that disagree with their own metadata leg, and
        // an as-is identity re-commit preserves that drift forever — the
        // oracle re-flags it every pass and existing brains never flip. The
        // metadata leg is the authority (denormalized fields are its
        // projections, per the field-addressing law); nothing degrades: the
        // floats ride through, adjacency residue has its own rebuild path.
        const wrapper =
          raw.vector !== null && typeof raw.vector === 'object' && !Array.isArray(raw.vector)
            ? (raw.vector as Record<string, unknown>)
            : null
        const vector = Array.isArray(raw.vector)
          ? (raw.vector as number[])
          : Array.isArray(wrapper?.vector)
            ? (wrapper!.vector as number[])
            : []
        const lawWrapper = reconstructNounWrapper(m.id, raw.metadata, vector)
        const priorRaw = { metadata: raw.metadata, vector: raw.vector }
        await this.persistSingleOp({ nouns: [m.id] }, async (tx) => {
          tx.addOperation({
            name: 'BaselineLawShapeRewrite',
            execute: async () => {
              await this.storage.writeNounRaw(m.id, {
                metadata: raw.metadata,
                vector: lawWrapper
              })
              return async () => {
                await this.storage.writeNounRaw(m.id, priorRaw)
              }
            }
          })
        }, undefined, undefined, undefined, 'system:adoption-backfill')
      }
      const next = await this.runOracle({ listAll: true })
      // THE ONLY STOP: no progress. With uncapped listings both counts are
      // exact, so "not fewer mismatches than before" means the cure could
      // not express this divergence — refuse to spin, name it.
      if (next.verdict === 'red' && next.mismatches.length >= report.mismatches.length) {
        throw new Error(
          `adoptLogAuthority(): baseline backfill made no progress ` +
            `(${report.mismatches.length} → ${next.mismatches.length} mismatches; first: ` +
            `${next.mismatches[0]?.reason} on ${next.mismatches[0]?.id}) — refusing to loop. ` +
            `This is a divergence class the backfill cannot express; investigate.`
        )
      }
      report = next
    }
    if (passes > 0) {
      prodLog.info(
        `[Brainy] adoptLogAuthority: baseline backfill complete in ${passes} pass(es) — ` +
          `oracle ${report.verdict}, ${report.nounsChecked} noun(s) checked`
      )
    }

    this._logAuthority = await flipToLogAuthority(
      this.storage as unknown as LogAuthorityStorage,
      report
    )
    this.generationStore.setLogDurability('at-ack')
    // Fold-checkpoint chain, phase 2: the flip is recorded — open the stamp
    // gate so the next flush/close barrier writes the first checkpoint.
    this.generationStore.completeFoldCheckpointBootstrap()
    // ARM-AT-FLIP for the NON-FRESH brain (the chain refused the fresh-brain
    // arm because committed > 0): run one paged FULL canonical barrier now —
    // every live row's canonical bytes fsynced, bounded memory — then stamp
    // the first checkpoint. Without this, the chain could only arm at the
    // brain's first crash, and that crash paid a WHOLE-LOG fold: a production
    // brain hit exactly that on its first post-flip boot (a full-log
    // materializing fold, restarted three times mid-flight). Adoption already
    // pays O(N) oracle work; one more O(N) barrier founds bounded recovery
    // from minute zero.
    if (!this.generationStore.foldCheckpointChainArmed()) {
      const PAGE = 500
      let synced = 0
      prodLog.info(
        `[Brainy] adoptLogAuthority: founding the fold checkpoint — syncing every ` +
          `row's canonical bytes (paged; progress every 2000 rows)`
      )
      let offset = 0
      let cursor: string | undefined
      for (;;) {
        const page = await this.storage.getNouns({
          pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
        })
        const ids = page.items.map((i) => (i as { id: string }).id)
        if (ids.length > 0) {
          await this.storage.syncEntityCanonical?.(ids, [])
          synced += ids.length
          if (synced % 2000 < PAGE && synced >= 2000) {
            prodLog.info(`[Brainy] adoptLogAuthority: checkpoint founding — ${synced} rows synced`)
          }
        }
        if (page.hasMore && page.nextCursor) { cursor = page.nextCursor; offset += ids.length; continue }
        if (page.hasMore && !page.nextCursor) { offset += PAGE; continue }
        break
      }
      let vOffset = 0
      let vCursor: string | undefined
      for (;;) {
        const page = await this.storage.getVerbs({
          pagination: vCursor ? { limit: PAGE, cursor: vCursor } : { limit: PAGE, offset: vOffset }
        })
        const ids = page.items.map((i) => (i as { id: string }).id)
        if (ids.length > 0) {
          await this.storage.syncEntityCanonical?.([], ids)
          synced += ids.length
        }
        if (page.hasMore && page.nextCursor) { vCursor = page.nextCursor; vOffset += ids.length; continue }
        if (page.hasMore && !page.nextCursor) { vOffset += PAGE; continue }
        break
      }
      await this.generationStore.stampFoldCheckpointAfterFullBarrier()
    }
    return report
  }

  /**
   * @description THE ATTESTED PER-ID RECONCILE DOOR for the one divergence
   * class the adoption backfill refuses BY DESIGN: `log-live-canonical-absent`
   * — the log holds a live record for a row the canonical tree says does not
   * exist. The engine cannot tell a legitimate pre-log deletion (the log
   * missed the tombstone — the deferred-durability-era ack-window class) from
   * canonical LOSS (the log holds the only surviving copy); auto-curing would
   * silently destroy data in one of the two readings. A HUMAN attests which:
   *
   * - `attest: 'deleted'` — the row was legitimately deleted; mint the
   *   tombstone fact the log always lacked (canonical stays absent). The
   *   log's history keeps the old live record — as-of reads before the
   *   tombstone still see it.
   * - `attest: 'restore'` — canonical lost the row; fold the log's latest
   *   after-image back into canonical (both sides now agree it lives).
   *
   * Loud, narrated, single-row, and stamped `origin: 'system:reconcile'` on
   * both the tx-log entry and the commit fact. Refuses (typed) when the id's
   * log and canonical already agree, when `restore` is attested but the log
   * holds no record, and when canonical is PRESENT-but-different (that is
   * `state-differs` — `adoptLogAuthority()`'s backfill owns it).
   *
   * @param id - The single entity id to reconcile.
   * @param options.attest - The human's word on which reading is true.
   * @returns What was done and the generation that recorded it.
   * @throws When the divergence is not the attested class (nothing is written).
   */
  async reconcileLogDivergence(
    id: string,
    options: { attest: 'deleted' | 'restore' }
  ): Promise<{ reconciled: 'tombstoned' | 'restored'; id: string; generation: number }> {
    await this.ensureInitialized()
    this.assertWritable('reconcileLogDivergence')

    // Fold the log for THIS id (one scan; a rare operator door).
    const scan = this.scanFacts()
    if (!scan) {
      throw new Error('reconcileLogDivergence: this store has no fact log — nothing to reconcile against')
    }
    let logLatest: { tombstoned: boolean; record: { metadata: unknown; vector: unknown } | null } | null = null
    for await (const batch of scan.batches()) {
      for (const fact of batch.facts) {
        for (const op of fact.ops) {
          if (op.kind === 'noun' && op.id === id) {
            logLatest =
              op.record === null
                ? { tombstoned: true, record: null }
                : { tombstoned: false, record: { metadata: op.record.metadata, vector: op.record.vector } }
          }
        }
      }
    }
    const canonical = await this.storage.readNounRaw(id)
    const canonicalAbsent = canonical.metadata === null && canonical.vector === null

    // Only the log-live + canonical-absent shape passes; everything else
    // names its actual state and the door that owns it.
    if (!logLatest || logLatest.tombstoned) {
      throw new Error(
        `reconcileLogDivergence(${id}): the log's latest state is ` +
          `${logLatest ? 'a tombstone' : 'no record at all'} — there is no ` +
          `log-live-canonical-absent divergence here. If the oracle reports this id, ` +
          `re-run verifyLogAuthority() for the current class.`
      )
    }
    if (!canonicalAbsent) {
      throw new Error(
        `reconcileLogDivergence(${id}): canonical is PRESENT — this is not the ` +
          `log-live-canonical-absent class. If canonical differs from the log ` +
          `(state-differs), adoptLogAuthority()'s backfill cures it; nothing was written.`
      )
    }

    if (options.attest === 'deleted') {
      // Mint the tombstone fact the log always lacked. writeNounRaw with null
      // parts is an idempotent delete; the commit fact reads canonical back
      // after execute (absent) and records the tombstone.
      const receipt = await this.persistSingleOp(
        { nouns: [id] },
        async (tx) => {
          tx.addOperation({
            name: 'ReconcileTombstone',
            execute: async () => {
              await this.storage.writeNounRaw(id, { metadata: null, vector: null })
              return async () => {
                // Undo of an idempotent delete of an absent row: nothing.
              }
            }
          })
        },
        undefined,
        undefined,
        undefined,
        'system:reconcile'
      )
      prodLog.warn(
        `[Brainy] reconcileLogDivergence: ${id} attested DELETED — tombstone fact minted ` +
          `at generation ${receipt.generation}; the log now agrees the row is gone ` +
          `(its history keeps the earlier live record).`
      )
      return { reconciled: 'tombstoned', id, generation: receipt.generation! }
    }

    // attest: 'restore' — the log's copy is the survivor; fold it back.
    const record = logLatest.record!
    const receipt = await this.persistSingleOp(
      { nouns: [id] },
      async (tx) => {
        tx.addOperation({
          name: 'ReconcileRestore',
          execute: async () => {
            await this.storage.writeNounRaw(id, record)
            return async () => {
              await this.storage.writeNounRaw(id, { metadata: null, vector: null })
            }
          }
        })
      },
      undefined,
      undefined,
      undefined,
      'system:reconcile'
    )
    prodLog.warn(
      `[Brainy] reconcileLogDivergence: ${id} attested RESTORE — the log's latest ` +
        `after-image was folded back into canonical at generation ${receipt.generation}. ` +
        `Derived indexes reconcile at next open/repairIndex; the row serves from canonical now.`
    )
    return { reconciled: 'restored', id, generation: receipt.generation! }
  }

  /**
   * @description Read the reified transaction log — one entry per committed
   * generation, carrying the committed generation, the commit timestamp, and
   * the `meta` the transaction was submitted with. Entries are returned newest
   * first. Under Model-B EVERY write is its own generation, so single-operation
   * writes (`add`/`update`/`remove`/`relate`/…) appear here too (without `meta`
   * — transaction metadata is a `transact()`-only concept). `compactHistory()`
   * never rewrites the log — entries may reference generations whose
   * record-sets were already reclaimed.
   *
   * @param options - `from`/`to` (generation number or `Date`) bound the window
   *   to commits in `[from, to]` INCLUSIVE on both ends — a log window names
   *   commits, in deliberate contrast to `since()`'s EXCLUSIVE lower bound.
   *   `limit` caps the result and is applied LAST (after windowing), newest
   *   first. All omitted = the whole log. `from`/`to` resolve through the same
   *   reachability gate as `asOf()`.
   * @returns Committed transaction entries, newest first.
   * @throws RangeError when the resolved `from` generation is after `to`.
   * @example
   * await brain.transact([{ op: 'update', id, metadata: { v: 2 } }], { meta: { author: 'sync-job' } })
   * const [latest] = await brain.transactionLog({ limit: 1 })
   * latest.meta // { author: 'sync-job' }
   * // Commits in a window, newest first:
   * const window = await brain.transactionLog({ from: 10, to: 20 })
   */
  async transactionLog(options?: {
    from?: number | Date
    to?: number | Date
    limit?: number
  }): Promise<TxLogEntry[]> {
    await this.ensureInitialized()
    let entries = await this.generationStore.txLog() // oldest first

    if (options?.from !== undefined || options?.to !== undefined) {
      const fromGen =
        options?.from !== undefined
          ? (await this.resolveAsOfGeneration(options.from)).generation
          : 0
      const toGen =
        options?.to !== undefined
          ? (await this.resolveAsOfGeneration(options.to)).generation
          : this.generationStore.generation()
      if (fromGen > toGen) {
        throw new RangeError(
          `transactionLog(): resolved from-generation ${fromGen} is after to-generation ${toGen}`
        )
      }
      // Inclusive on BOTH ends (a log window names the commits it spans).
      entries = entries.filter((e) => e.generation >= fromGen && e.generation <= toGen)
    }

    entries.reverse() // newest first
    return options?.limit !== undefined ? entries.slice(0, options.limit) : entries
  }

  /**
   * @description Pin the CURRENT generation and return an immutable `Db`
   * view of it — O(1), no I/O. The returned view keeps serving the FULL
   * query surface at exactly this state no matter what commits afterwards:
   * `get()` and metadata-level `find()`/`related()` resolve from immutable
   * generation records at no extra cost, while index-accelerated queries
   * (semantic/vector search, graph traversal, cursors, aggregation) are
   * served by an at-generation index materialization built lazily on first
   * use — O(n at G) time and memory once per `Db`, freed on `release()`
   * (a native `VersionedIndexProvider` serves the same reads from retained
   * segments without rebuild).
   *
   * Call `db.release()` when done — pins gate `compactHistory()`. A
   * `FinalizationRegistry` backstop releases leaked pins at GC time, but
   * explicit release is what makes compaction deterministic.
   *
   * @returns A `Db` pinned at the current generation.
   * @throws Error when called before `init()` completes (pinning is
   *   synchronous, so it cannot await initialization).
   * @example
   * const db = brain.now()
   * await brain.transact([{ op: 'update', id, metadata: { v: 2 } }])
   * await db.get(id)    // still sees v: 1
   * await brain.get(id) // sees v: 2
   * await db.release()
   */
  now(): Db<T> {
    this.assertGenerationStoreReady('now')
    return this.createPinnedDb({
      generation: this.generationStore.generation(),
      timestamp: Date.now()
    })
  }

  /**
   * @description Serialize part or all of the brain into a portable `PortableGraph`
   * document — sugar for `brain.now().export(selector, options)` (the current
   * generation). For a past generation use `(await brain.asOf(g)).export(...)`;
   * for a speculative state use `brain.now().with(ops).export(...)`.
   *
   * Restore a `PortableGraph` with {@link Brainy.import} (it dispatches a backup to the
   * graph round-trip; a file/buffer to ingestion). Distinct from `db.persist()`
   * (native whole-brain snapshot with generation history).
   *
   * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}.
   * @param options - HOW to export (vectors / VFS bytes / edge policy). See {@link ExportOptions}.
   * @returns A versioned, portable `PortableGraph` document.
   * @example
   * const graph = await brain.export({ ids }, { includeVectors: true })
   */
  async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise<PortableGraph> {
    return this.now().export(selector, options)
  }

  /**
   * @description Execute a declarative operation batch atomically: either
   * every operation applies and the store advances exactly ONE generation,
   * or none apply and the store is byte-identical to its pre-transaction
   * state. The semantics of each operation mirror the corresponding
   * single-operation method (`add`/`update`/`remove`/`relate`/`unrelate`),
   * including validation, subtype enforcement, per-entity `ifRev` CAS,
   * relationship deduplication, and delete cascades. Operations may
   * reference ids created by earlier operations of the same batch.
   *
   * Durability protocol (see `src/db/generationStore.ts`): before-images of
   * every touched id are staged and fsynced, the batch executes through the
   * TransactionManager, and the atomic rename of `_system/manifest.json` is
   * the commit point — a crash anywhere before the rename is rolled back to
   * the exact pre-transaction bytes on the next open.
   *
   * Isolation: concurrent `transact()` calls commit serially
   * (snapshot-isolated batches). For strict lost-update protection across a
   * read-modify-write cycle, pass `ifAtGeneration` (whole-store CAS) or use
   * per-entity `ifRev` on update operations.
   *
   * @param ops - Declarative operations: `{ op: 'add', ... }`,
   *   `{ op: 'update', ... }`, `{ op: 'remove', id }`,
   *   `{ op: 'relate', ... }`, `{ op: 'unrelate', id }`.
   * @param options - `meta` (reified transaction metadata, recorded in
   *   `_system/tx-log.jsonl`) and `ifAtGeneration` (whole-store CAS).
   * @returns A `Db` pinned at the freshly committed generation, carrying a
   *   `receipt` with the resolved id per input operation.
   * @throws GenerationConflictError when `ifAtGeneration` does not match the
   *   store's current generation.
   * @throws RevisionConflictError when an update operation's `ifRev` does
   *   not match the entity's persisted revision (whole batch rejected).
   * @example
   * const db = await brain.transact([
   *   { op: 'add', id: aId, type: NounType.Person, subtype: 'employee', data: 'Ada' },
   *   { op: 'add', id: bId, type: NounType.Project, subtype: 'milestone', data: 'Apollo' },
   *   { op: 'relate', from: aId, to: bId, type: VerbType.ParticipatesIn, subtype: 'assignment' }
   * ], { meta: { author: 'import-job-42' } })
   * db.receipt.ids // [aId, bId, relationshipId]
   */
  async transact(ops: TxOperation<T>[], options?: TransactOptions): Promise<Db<T>> {
    this.assertWritable('transact')
    await this.ensureInitialized()

    if (!Array.isArray(ops) || ops.length === 0) {
      throw new Error('transact(): provide a non-empty array of transaction operations')
    }

    // Commit any buffered single-op generations FIRST so this transaction's
    // generation lands strictly after them and the committed-generation
    // sequence stays gap-free (single-ops reserve generations eagerly).
    await this.generationStore.flushPendingSingleOps()

    // Early CAS check — avoids expensive planning (embedding) on a stale
    // expectation. The generation store re-checks authoritatively under the
    // commit mutex; this one is purely a fast-fail.
    if (
      options?.ifAtGeneration !== undefined &&
      options.ifAtGeneration !== this.generationStore.generation()
    ) {
      throw new GenerationConflictError(
        options.ifAtGeneration,
        this.generationStore.generation()
      )
    }

    const plan = await this.planTransact(ops)

    // Authoritative per-op ifRev CAS, run by the generation store UNDER the
    // commit mutex against the just-read before-images — atomic with the
    // apply (the planning-time checks above are fast-fails that can
    // interleave with concurrent writers). Any conflict rejects the WHOLE
    // batch before anything is staged or applied. Sequencing rules:
    //   - runningRev tracks multiple updates to the SAME entity in one batch
    //     (each op CASes against — and bumps — its predecessor's result).
    //   - An entity the batch itself creates (null before-image, forward-ref
    //     add+update) keeps its plan-sequenced stamps: no concurrent writer
    //     can have moved a rev that did not exist, so plan-time was exact.
    const casPrecommit = (before: CommitBeforeImages): void => {
      const runningRev = new Map<string, number>()
      for (const cas of plan.casUpdates) {
        // An id this batch adds owns its revision baseline (the add restamps
        // `_rev: 1` even on overwrite) — plan-time sequencing is exact, no
        // concurrent writer can move a baseline the batch itself sets.
        if (plan.createdNouns.has(cas.id)) {
          continue
        }
        const beforeMeta = before.nouns.get(cas.id)?.metadata as
          | { _rev?: unknown }
          | null
          | undefined
        if (!beforeMeta && !runningRev.has(cas.id)) {
          // A pre-existing entity a concurrent remove() deleted after
          // planning: a CAS caller gets the truth; a plain update keeps its
          // last-writer-wins re-create semantics (plan-stamped rev).
          if (typeof cas.ifRev === 'number') {
            throw new EntityNotFoundError(cas.id)
          }
          continue
        }
        const base =
          runningRev.get(cas.id) ??
          (typeof beforeMeta?._rev === 'number' ? beforeMeta._rev : 1)
        if (typeof cas.ifRev === 'number' && cas.ifRev !== base) {
          throw new RevisionConflictError(cas.id, cas.ifRev, base)
        }
        const next = base + 1
        cas.updatedMetadata._rev = next
        runningRev.set(cas.id, next)
      }
    }

    let generation: number
    let timestamp: number
    try {
      ;({ generation, timestamp } = await this.generationStore.commitTransaction({
        touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs },
        meta: options?.meta,
        ifAtGeneration: options?.ifAtGeneration,
        precommit: casPrecommit,
        ...(plan.markerRecords.length > 0 ? { records: plan.markerRecords } : {}),
        execute: async () => {
          await this.transactionManager.executeTransaction(
            async (tx) => {
              for (const operation of plan.operations) {
                tx.addOperation(operation)
              }
            },
            // Budget scales with the batch (or the caller's explicit
            // timeoutMs): a flat 30s cap silently limited honest bulk work
            // to ~15 ops on network disks (~2s/op measured in the field).
            {
              timeout: transactTimeoutBudget(
                plan.operations.length,
                options?.timeoutMs,
                this.config.transactionBudgetFloorMs
              )
            }
          )
        }
      }))
    } catch (err) {
      // A batch whose rollback failed and left canonical records unreconciled
      // quarantines writes until repairIndex() reconciles and lifts it.
      if (err instanceof StoreInconsistentError) this.storeInconsistency = err
      throw err
    }

    // Aggregation-index maintenance: derived data, applied after the commit
    // point — exactly where the single-operation methods apply it.
    for (const hook of plan.postCommit) {
      hook()
    }

    // Leg D — vectored-ledger decrements for this batch's unvector-door
    // updates (see planTxUpdate's matching comment), applied after the
    // commit point and properly awaited (unlike `postCommit`'s synchronous
    // fire-and-forget hooks) — each is the same sanctioned hook
    // unvectorNounForRootMigration() uses.
    for (const id of plan.vectorUnlands) {
      await this.storage.noteVectorUnlanded?.(id)
    }

    // Change feed: the batch's events share its single committed generation.
    // A rejected batch throws at commitTransaction and never reaches here.
    this.emitCommitted(plan.changeEvents, undefined, generation, timestamp)

    this.noteWriteForPersistence()
    const receipt: TransactReceipt = { generation, timestamp, ids: plan.ids }
    return this.createPinnedDb({ generation, timestamp, receipt })
  }

  /**
   * @description Open an immutable `Db` view of PAST state:
   *
   * - **`number`** — a generation: pins it on this store; reads resolve
   *   through the generational record layer. Under Model-B every write is its
   *   own generation, so single-operation writes are individually addressable
   *   too (a pin always freezes against later single-op writes).
   * - **`Date`** — a wall-clock instant: resolved via the transaction log to
   *   the newest generation committed at or before it, then pinned as above.
   * - **`string`** — a snapshot directory previously produced by
   *   `db.persist(path)`: opened as a self-contained read-only store
   *   (equivalent to {@link Brainy.load}) with the FULL query surface,
   *   including vector search.
   *
   * Release the returned `Db` when done.
   *
   * @param target - Generation number, `Date`, or snapshot directory path.
   * @param options - `exclusive: true` pins the generation immediately BEFORE the
   *   one `target` resolves to (strict-before): `asOf(g, { exclusive: true })` is
   *   the state at `g − 1`; strict-before the first commit pins generation 0 (the
   *   empty pre-commit state), never a `RangeError`. Ignored for the snapshot
   *   (string) form.
   * @returns A `Db` pinned at the resolved state.
   * @throws GenerationCompactedError when the generation's records were
   *   reclaimed by `compactHistory()`.
   * @throws RangeError for negative or future generations.
   * @example
   * const yesterday = await brain.asOf(new Date(Date.now() - 86_400_000))
   * const atGen42 = await brain.asOf(42)
   * const beforeGen42 = await brain.asOf(42, { exclusive: true }) // state at gen 41
   * const fromSnapshot = await brain.asOf('/backups/2026-06-01')
   */
  async asOf(
    target: number | Date | string,
    options?: { exclusive?: boolean }
  ): Promise<Db<T>> {
    await this.ensureInitialized()

    if (typeof target === 'string') {
      const stat = await fs.promises.stat(target).catch(() => null)
      if (!stat || !stat.isDirectory()) {
        throw new Error(
          `asOf(): '${target}' is not a snapshot directory. Pass a generation number, ` +
            `a Date, or a directory created by db.persist(path).`
        )
      }
      return Brainy.load<T>(target)
    }

    const { generation, timestamp } = await this.resolveAsOfGeneration(target, options)
    return this.createPinnedDb({ generation, timestamp })
  }

  /**
   * @description The shared `generation | Date` → `{ generation, timestamp }`
   * resolver used by every gen/Date-accepting temporal verb (`asOf`, `since`,
   * `diff`, `transactionLog`) so reachability, `RangeError`, and `Date` semantics
   * stay identical across all of them. The inclusive path is byte-for-byte the
   * pre-extraction `asOf` logic; `exclusive` pins the generation immediately
   * before the resolved one (clamped at 0, never a `RangeError`). All resolved
   * generations pass through `assertReachable` (so a compacted target throws
   * {@link GenerationCompactedError}); `history()` deliberately does NOT use this
   * resolver — it truncates below the horizon rather than throwing.
   * @param target - A generation number or a `Date`.
   * @param options - `exclusive: true` for strict-before (resolved − 1).
   */
  private async resolveAsOfGeneration(
    target: number | Date,
    options?: { exclusive?: boolean }
  ): Promise<{ generation: number; timestamp: number }> {
    const exclusive = options?.exclusive === true
    let generation: number
    let timestamp: number
    if (target instanceof Date) {
      const resolved = await this.generationStore.resolveTimestamp(target.getTime())
      if (exclusive) {
        generation = Math.max(0, resolved.generation - 1)
        this.generationStore.assertReachable(generation)
        timestamp =
          (await this.generationStore.commitTimestampAtOrBefore(generation)) ?? target.getTime()
      } else {
        this.generationStore.assertReachable(resolved.generation)
        generation = resolved.generation
        timestamp = resolved.entry?.timestamp ?? target.getTime()
      }
    } else {
      generation = exclusive ? Math.max(0, target - 1) : target
      this.generationStore.assertReachable(generation)
      timestamp =
        (await this.generationStore.commitTimestampAtOrBefore(generation)) ?? Date.now()
    }
    return { generation, timestamp }
  }

  /**
   * @description Compare two states of the brain and report what CHANGED between
   * them — entities/relationships `added`, `removed`, or `modified`, each split by
   * kind. Unlike a raw touched-id list, `diff` EARNS its name: the candidate set is
   * ONLY the ids a committed transaction touched in the interval, and each is then
   * resolved at BOTH endpoints and classified by existence and a stable value
   * comparison. An id touched within the interval but whose endpoint states are
   * identical (e.g. changed then reverted) appears in NONE of the three buckets.
   *
   * Orientation is `a → b`: `added` = exists at `b` not `a`; `removed` = exists at
   * `a` not `b`; `modified` = exists at both with a changed stored value
   * (key-order-insensitive). Endpoints may be passed in either order.
   *
   * @param a - The "before" state: a generation number, a `Date`, or a `Db`.
   * @param b - The "after" state: a generation number, a `Date`, or a `Db`.
   * @returns A {@link DiffResult} (each bucket's ids sorted).
   * @throws GenerationCompactedError when a number/`Date` endpoint is below the horizon.
   * @example
   * const before = brain.generation()
   * await brain.transact(ops)
   * const d = await brain.diff(before, brain.generation())
   * d.added.nouns      // ids created by the transaction
   * d.modified.nouns   // ids whose stored value actually changed
   */
  async diff(a: number | Date | Db<T>, b: number | Date | Db<T>): Promise<DiffResult> {
    await this.ensureInitialized()
    const gA = await this.resolveDiffEndpoint(a)
    const gB = await this.resolveDiffEndpoint(b)

    const result: DiffResult = {
      added: { nouns: [], verbs: [] },
      removed: { nouns: [], verbs: [] },
      modified: { nouns: [], verbs: [] },
      fromGeneration: gA,
      toGeneration: gB
    }

    // Candidate set: ONLY the ids touched in the interval (never all ids) — this
    // is what lets diff classify rather than dump raw changed-ids.
    const gLow = Math.min(gA, gB)
    const gHigh = Math.max(gA, gB)
    if (gLow === gHigh) return result // same state → no changes
    const changed = await this.generationStore.changedBetween(gLow, gHigh)

    for (const kind of ['noun', 'verb'] as const) {
      const ids = kind === 'noun' ? changed.nouns : changed.verbs
      for (const id of ids) {
        const stateA = await this.recordStateAt(kind, id, gA)
        const stateB = await this.recordStateAt(kind, id, gB)
        const bucket = (name: 'added' | 'removed' | 'modified') =>
          kind === 'noun' ? result[name].nouns : result[name].verbs
        if (!stateA.exists && stateB.exists) bucket('added').push(id)
        else if (stateA.exists && !stateB.exists) bucket('removed').push(id)
        else if (
          stateA.exists &&
          stateB.exists &&
          !stableDeepEqual(stateA.metadata, stateB.metadata)
        ) {
          bucket('modified').push(id)
        }
        // both absent, or both exist and identical → in NO bucket (earns the name)
      }
    }

    for (const name of ['added', 'removed', 'modified'] as const) {
      result[name].nouns.sort()
      result[name].verbs.sort()
    }
    return result
  }

  /**
   * @description Every distinct version of ONE entity or relationship over a
   * generation range, oldest → newest — the per-id companion to `asOf()`. Each
   * {@link HistoryVersion} ties to the trusted `asOf()` path: its `value` equals
   * `(await brain.asOf(version.generation)).get(id)`. Consecutive identical states
   * are collapsed, and a leading "did not exist yet" baseline is dropped, so the
   * first version is the id's first real state in the range and a `null` value
   * marks a removal.
   *
   * Range defaults to `[horizon, currentGeneration]`. Unlike `diff`/`since`, a
   * `from` below the compaction horizon is TRUNCATED to the horizon, not thrown —
   * history is best-effort over the records that survive.
   *
   * @param id - The entity or relationship id (kind auto-detected).
   * @param options - `from`/`to` (generation number or `Date`) bound the range.
   * @returns An {@link EntityHistory} (`versions` oldest first; empty if the id is
   *   unknown over the range).
   * @throws Error if `id` resolves in BOTH the entity and relationship spaces.
   * @example
   * const h = await brain.history(invoiceId)
   * h.versions.map(v => v.value?.metadata?.status) // the status at each change
   */
  async history(
    id: string,
    options?: { from?: number | Date; to?: number | Date }
  ): Promise<EntityHistory<T>> {
    await this.ensureInitialized()
    const store = this.generationStore
    const horizon = store.horizon()
    const current = store.generation()

    // history TRUNCATES below the horizon (never throws): resolve raw, then clamp.
    const rawFrom =
      options?.from !== undefined ? await this.resolveRawGeneration(options.from) : horizon
    const rawTo =
      options?.to !== undefined ? await this.resolveRawGeneration(options.to) : current
    const fromGen = Math.min(Math.max(rawFrom, horizon), current)
    const toGen = Math.min(Math.max(rawTo, 0), current)

    const kind = await this.detectIdKind(id, horizon, current)
    if (kind === null || fromGen > toGen) {
      return { id, kind: kind ?? 'noun', versions: [] }
    }

    // Snapshot generations: the range start (state as-of fromGen) plus every
    // committed change-point in (fromGen, toGen].
    const touches = await store.generationsTouching(kind, id, fromGen, toGen)
    const snapshotGens = [fromGen, ...touches]

    const versions: HistoryVersion<T>[] = []
    let prev: { exists: boolean; metadata: any } | null = null
    for (const g of snapshotGens) {
      const state = await this.recordStateAt(kind, id, g)
      if (
        prev !== null &&
        prev.exists === state.exists &&
        stableDeepEqual(prev.metadata, state.metadata)
      ) {
        continue // collapse consecutive identical states
      }
      prev = { exists: state.exists, metadata: state.metadata }
      const timestamp = (await store.commitTimestampAtOrBefore(g)) ?? 0
      let value: Entity<T> | Relation<T> | null = null
      if (state.exists) {
        value =
          kind === 'noun'
            ? await this.entityFromGenerationRecord(
                id,
                { metadata: state.metadata, vector: state.vector },
                false
              )
            : this.relationFromGenerationRecord(id, {
                metadata: state.metadata,
                vector: state.vector
              })
      }
      versions.push({ generation: g, timestamp, value })
    }

    // Drop a leading "did not exist yet" baseline so history starts at the id's
    // first real state (internal/trailing nulls = real removals stay).
    while (versions.length > 0 && versions[0].value === null) versions.shift()
    return { id, kind, versions }
  }

  /** Resolve a `Db | generation | Date` diff endpoint to a concrete generation. */
  private async resolveDiffEndpoint(endpoint: number | Date | Db<T>): Promise<number> {
    if (endpoint instanceof Db) {
      if (!endpoint.belongsToStore(this.generationStore)) {
        throw new Error('diff(): a Db endpoint must come from this same Brainy instance')
      }
      return endpoint.generation
    }
    return (await this.resolveAsOfGeneration(endpoint)).generation
  }

  /**
   * Resolve a `generation | Date` to a raw generation number WITHOUT the
   * reachability assertion — for `history()`, which truncates below the horizon
   * rather than throwing.
   */
  private async resolveRawGeneration(target: number | Date): Promise<number> {
    if (target instanceof Date) {
      return (await this.generationStore.resolveTimestamp(target.getTime())).generation
    }
    return target
  }

  /**
   * The stored state of `id` at pinned generation `gen`, normalizing `resolveAt`'s
   * three sources to `{ exists, metadata, vector }`. A `'current'` source means the
   * live storage state IS the state at `gen`, so it reads the live metadata.
   */
  private async recordStateAt(
    kind: 'noun' | 'verb',
    id: string,
    gen: number
  ): Promise<{ exists: boolean; metadata: any; vector: any }> {
    const r = await this.generationStore.resolveAt(kind, id, gen)
    if (r.source === 'absent') return { exists: false, metadata: null, vector: null }
    if (r.source === 'record') return { exists: true, metadata: r.metadata, vector: r.vector }
    const metadata =
      kind === 'noun'
        ? await this.storage.getNounMetadata(id)
        : await this.storage.getVerbMetadata(id)
    return { exists: metadata !== null, metadata, vector: null }
  }

  /**
   * Auto-detect whether `id` is an entity (`'noun'`) or relationship (`'verb'`) by
   * its presence live and in the generation deltas over `(fromGen, toGen]`. Returns
   * `null` for an unknown id; throws on the (UUID-space) collision where the id
   * resolves in both spaces.
   */
  private async detectIdKind(
    id: string,
    fromGen: number,
    toGen: number
  ): Promise<'noun' | 'verb' | null> {
    const liveNoun = (await this.storage.getNounMetadata(id)) !== null
    const liveVerb = (await this.storage.getVerbMetadata(id)) !== null
    const nounTouched =
      (await this.generationStore.generationsTouching('noun', id, fromGen, toGen)).length > 0
    const verbTouched =
      (await this.generationStore.generationsTouching('verb', id, fromGen, toGen)).length > 0
    const isNoun = liveNoun || nounTouched
    const isVerb = liveVerb || verbTouched
    if (isNoun && isVerb) {
      throw new Error(
        `history(): id '${id}' resolves in BOTH the entity and relationship spaces — ` +
          `ambiguous (should not happen with UUID ids).`
      )
    }
    if (isNoun) return 'noun'
    if (isVerb) return 'verb'
    return null
  }

  /**
   * @description Reclaim the immutable generation record-sets that no
   * retention rule and no live pin protects. Records are what serve
   * historical reads (`asOf()`, pinned `Db` values); compaction trades that
   * history for disk space. Generations at or below the resulting horizon
   * become unreachable (`asOf()` throws `GenerationCompactedError`).
   *
   * Safety invariant: a record-set is never removed while any live `Db` pin
   * could need it — pinned reads stay correct across compaction, always.
   * Note that `transact()` returns a PINNED `Db`: release views you do not
   * keep (the GC backstop eventually releases leaked ones, but until then
   * compaction retains the history they protect).
   *
   * @param options - Explicit retention CAPS (`maxGenerations` / `maxAge` /
   *   `maxBytes`). Reclaim the oldest unpinned generations while ANY supplied
   *   cap is exceeded. Neither = reclaim everything unpinned.
   * @returns Count of reclaimed record-sets and the new horizon.
   * @example
   * await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 86_400_000 })
   */
  async compactHistory(options?: CompactHistoryOptions): Promise<CompactHistoryResult> {
    this.assertWritable('compactHistory')
    await this.ensureInitialized()
    // Persist buffered single-op generations first so compaction reasons over a
    // complete on-disk history (the pending tier is never itself reclaimable).
    await this.generationStore.flushPendingSingleOps()
    return this.generationStore.compact(options)
  }

  /**
   * @description Repack cold generation history into sealed segments —
   * re-representation, never deletion: every record and delta stays readable
   * (`asOf()` unchanged); the physical file count drops by orders of
   * magnitude. Runs automatically (time-bounded) at `close()`; call this for
   * explicit maintenance windows on long-lived writers. The ONLY history
   * transform permitted under the archival profile (`retention: 'all'`).
   * @param options - `timeBudgetMs` bounds the pass (early stop = consistent
   *   prefix, next pass resumes); `batchGenerations` sizes each fold.
   * @returns Folded generation count and segments created.
   */
  async repackHistory(options?: {
    timeBudgetMs?: number
    batchGenerations?: number
  }): Promise<{ foldedGenerations: number; segmentsCreated: number }> {
    this.assertWritable('repackHistory')
    await this.ensureInitialized()
    await this.generationStore.flushPendingSingleOps()
    return this.generationStore.repackHistory(options)
  }

  /**
   * @description Read-only generational-history footprint for fleet audits:
   * generation count, total on-disk bytes, generation/timestamp range, the
   * compaction horizon, and the retention policy in force. Touches no data and
   * changes nothing. First call pays one walk over committed deltas to seed
   * the running byte total (subsequent calls — and every adaptive retention
   * check — are then O(1)).
   *
   * A pool operator's exposure check is one call per brain:
   * @example
   * const stats = await brain.historyStats()
   * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`)
   */
  async historyStats(): Promise<HistoryStats> {
    await this.ensureInitialized()
    const stats = await this.generationStore.historyStats()
    const policy = this.resolveRetentionPolicy()
    return {
      ...stats,
      retentionMode: policy.mode,
      effectiveBudgetBytes:
        policy.mode === 'adaptive'
          ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes)
          : null
    }
  }

  /**
   * @description A deterministic content digest of the generation log through
   * `g` (D8 — gate-to-generation provenance): identical history produces the
   * identical digest on any machine; divergence produces a different one.
   * Release gates and suite verdicts pin `{generation, digest}` and verify
   * both at execution time instead of pinning a git commit. O(segments +
   * live-tier window), never O(all generations). Throws `RangeError` out of
   * range and `GenerationCompactedError` below the horizon — a gate can
   * never silently pin reclaimed history.
   * @example
   * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) }
   */
  async generationDigest(g: number): Promise<string> {
    await this.ensureInitialized()
    await this.generationStore.flushPendingSingleOps()
    return this.generationStore.generationDigest(g)
  }

  /**
   * @description Drive the adaptive retention byte budget at runtime — the
   * settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
   * which fair-shares one box-wide history budget across co-located instances)
   * pushes whenever the box's free-resource picture changes. Takes effect at the
   * next auto-compaction (flush/close). Only meaningful while `retention` is
   * adaptive (unset or `'adaptive'`); ignored under `'all'` or explicit caps.
   * @param bytes - The new per-brain history byte budget (non-negative).
   * @example brain.setRetentionBudget(2 * 1024 ** 3) // 2 GiB for this brain
   */
  setRetentionBudget(bytes: number): void {
    if (!Number.isFinite(bytes) || bytes < 0) {
      throw new RangeError(`setRetentionBudget(): bytes must be a non-negative finite number (got ${bytes})`)
    }
    this._retentionBudgetBytes = bytes
  }

  /**
   * @description Resolve the effective `retention` policy from `config.retention`.
   * The single read site for the policy {@link autoCompactHistory} applies.
   *
   * - `'all'` → `{ mode: 'all' }` (never reclaim history).
   * - unset / `'adaptive'` → `{ mode: 'adaptive' }` (budget-driven; the budget
   *   is `setRetentionBudget()`/`budgetBytes` when set, else a local probe).
   * - `{ … }` → `{ mode: 'explicit', maxGenerations?, maxAge?, maxBytes? }`.
   *
   * `autoCompact` defaults to `true` in every mode.
   * @returns The normalized retention policy.
   */
  private resolveRetentionPolicy(): {
    mode: 'all' | 'adaptive' | 'explicit'
    maxGenerations?: number
    maxAge?: number
    maxBytes?: number
    budgetBytes?: number
    autoCompact: boolean
  } {
    const retention = this.config.retention
    if (retention === 'all') {
      return { mode: 'all', autoCompact: true }
    }
    if (retention === undefined || retention === 'adaptive') {
      return { mode: 'adaptive', autoCompact: true }
    }
    return {
      mode: 'explicit',
      maxGenerations: retention.maxGenerations,
      maxAge: retention.maxAge,
      maxBytes: retention.maxBytes,
      budgetBytes: retention.budgetBytes,
      autoCompact: retention.autoCompact ?? true
    }
  }

  /**
   * @description Compute the adaptive history byte budget for this brain: the
   * coordinator-driven value when present (`setRetentionBudget()` /
   * `config.retention.budgetBytes` — e.g. cor's fair-shared box budget), else a
   * conservative LOCAL probe of free resources. Standalone brainy must not be
   * cor-dependent, so the fallback uses `os.freemem()` (and is intentionally
   * generous — a single instance owning the box can keep a lot of history).
   * @returns The byte budget; `Infinity` only if no signal is available.
   */
  private adaptiveHistoryBudgetBytes(driven?: number): number {
    if (driven !== undefined) return driven
    if (this._retentionBudgetBytes !== undefined) return this._retentionBudgetBytes
    // Local single-instance probe: allot a quarter of currently-free RAM to
    // history. Bounded, zero-config, and never reclaims pinned generations.
    try {
      const free = os.freemem()
      if (Number.isFinite(free) && free > 0) return Math.floor(free / 4)
    } catch {
      // os unavailable (exotic runtime) — fall through to unbounded.
    }
    return Infinity
  }

  /**
   * @description Run history compaction under the resolved `retention` policy
   * when `autoCompact` is on (the default). Invoked from `close()` ONLY
   * (8.9.0) — flush() is durability work and never pays maintenance costs; a
   * production deployment measured reclaim-on-flush blocking single writes
   * for 25-191s under memory pressure. Long-lived writers that never close
   * accumulate history until their next explicit `compactHistory()` — the
   * documented trade: predictable writes, explicit maintenance.
   *
   * - `'all'` → returns without reclaiming (index compaction for speed still
   *   runs elsewhere; history is decoupled and kept).
   * - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget.
   * - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps.
   *
   * Every auto pass is TIME-BOUNDED ({@link CLOSE_COMPACTION_BUDGET_MS}) so a
   * large backlog can never stall a clean shutdown; the next pass resumes.
   * Read-only instances and an explicit `autoCompact: false` skip silently.
   * Pinned generations are never reclaimed ({@link GenerationStore.compact}).
   * Failures are logged and swallowed — compaction is housekeeping and must
   * never fail a clean shutdown.
   */
  private async autoCompactHistory(): Promise<void> {
    // Nothing to compact on a read-only instance or before init wired up the
    // generation store (close() can run defensively on an uninitialized brain).
    if (this.isReadOnly || !this.generationStore) {
      return
    }
    const policy = this.resolveRetentionPolicy()
    if (!policy.autoCompact || policy.mode === 'all') {
      return
    }
    try {
      if (policy.mode === 'adaptive') {
        const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes)
        if (budget === Infinity) return // no pressure signal → keep everything this pass
        await this.generationStore.compact({
          maxBytes: budget,
          timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS
        })
      } else {
        await this.generationStore.compact({
          maxGenerations: policy.maxGenerations,
          maxAge: policy.maxAge,
          maxBytes: policy.maxBytes,
          timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS
        })
      }
    } catch (error) {
      console.warn(
        `Auto-compaction of generational history failed (non-fatal): ${
          error instanceof Error ? error.message : String(error)
        }`
      )
    }
  }

  /**
   * @description Replace this store's ENTIRE state from a snapshot directory
   * previously produced by `db.persist(path)`. Destructive: current
   * entities, relationships, indexes, and history records are all replaced
   * by the snapshot's. The generation counter is floored at its pre-restore
   * value, so generation numbers observed before the restore are never
   * reissued.
   *
   * Live `Db` values pinned before a restore are NOT remapped — their
   * snapshot-isolation guarantee does not survive a wholesale state
   * replacement. Release them first; a warning is logged when live pins
   * exist.
   *
   * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a
   * staging area BEFORE any live data is touched (all-zero blocks stay
   * holes, so a sparse mmap store restores at its true allocated size — not
   * its apparent size). Any copy failure, ENOSPC included, removes only the
   * staging and throws; the live store is untouched. Only after the copy
   * succeeds does an atomic per-entry swap move it into place; a crash
   * mid-swap completes FORWARD on the next open.
   *
   * @param path - Snapshot directory to restore from.
   * @param options - Must be `{ confirm: true }` — an explicit acknowledgment
   *   that current state is destroyed.
   * @throws Error when `confirm` is not `true`, or `path` is not a snapshot
   *   directory.
   * @example
   * await brain.restore('/backups/2026-06-01', { confirm: true })
   */
  async restore(path: string, options: { confirm: boolean }): Promise<void> {
    this.assertWritable('restore')
    await this.ensureInitialized()

    if (options?.confirm !== true) {
      throw new Error(
        `restore() replaces the store's entire current state with the snapshot at ` +
          `'${path}'. Pass { confirm: true } to proceed.`
      )
    }

    const pinCount = this.generationStore.activePinCount()
    if (pinCount > 0) {
      prodLog.warn(
        `[Brainy] restore() with ${pinCount} live Db pin(s): pinned views do not ` +
          `survive a restore — release Db values before restoring.`
      )
    }

    const floorGeneration = this.generationStore.generation()
    // The swap runs inside the generation store's exclusive section: pending
    // flush timers are disarmed and buffers discarded BEFORE any directory is
    // removed, so a background flush can never write into `_system/` mid-swap
    // (the ENOTEMPTY race a checkpoint stamp once hit).
    await this.generationStore.runStateReplacement(() =>
      this.storage.restoreFromDirectory(path)
    )
    await this.generationStore.reopenAfterRestore(floorGeneration)

    // If the entity-id mapper is a NATIVE provider with a `rebuild()`, reload it
    // from the restored snapshot FIRST. The graph index resolves every verb
    // endpoint (sourceId/targetId → stable int) through this mapper, so a native
    // adjacency (keyed on those ints) needs the mapper to reflect the snapshot's
    // assignments BEFORE graphIndex.rebuild() — otherwise edges resolve to
    // stale/missing ints and are silently dropped (CTX-BR-RESTORE-REBUILD). The
    // JS mapper has no `rebuild()` and needs no reload: MetadataIndex.rebuild()
    // re-derives it from the restored entities via append-only getOrAssign,
    // consistently with the bitmaps it builds.
    await this.hydrateIdMapperForGraphRebuild()

    // Every in-memory index is now stale — rebuild all three from the restored
    // canonical records (each rebuild clears its own state first). The graph
    // rebuild resolves endpoints via the (now-reloaded, for native) mapper.
    await Promise.all([
      this.metadataIndex.rebuild(),
      this.index.rebuild(),
      this.graphIndex.rebuild()
    ])

    // Change feed: a restore is a wholesale raw-state replacement, not a
    // per-record commit — one store-level event tells subscribers to refetch.
    this._changeFeed.emit([
      { kind: 'store', op: 'restore', timestamp: Date.now() }
    ])
  }

  /**
   * @description Construct AND initialize a `Brainy` instance in one call —
   * `new Brainy(config)` + `await init()`.
   *
   * @param config - Standard `BrainyConfig`.
   * @returns The initialized instance.
   * @example
   * const brain = await Brainy.open({ storage: { type: 'filesystem', path: './data' } })
   */
  static async open<T = any>(config?: BrainyConfig): Promise<Brainy<T>> {
    const brain = new Brainy<T>(config)
    await brain.init()
    return brain
  }

  /**
   * @description Open a persisted snapshot directory (from `db.persist()`)
   * as a self-contained READ-ONLY store and return it as a `Db`. The
   * snapshot's indexes load from its own files, so the full query surface —
   * including vector search — works at the snapshot's generation. Releasing
   * the returned `Db` closes the underlying read-only instance.
   *
   * @param path - Snapshot directory produced by `db.persist(path)`.
   * @returns A `Db` over the snapshot (release it to free resources).
   * @example
   * const db = await Brainy.load('/backups/2026-06-01')
   * const hits = await db.find({ query: 'quarterly invoices' })
   * await db.release()
   */
  static async load<T = any>(path: string): Promise<Db<T>> {
    const stat = await fs.promises.stat(path).catch(() => null)
    if (!stat || !stat.isDirectory()) {
      throw new Error(
        `Brainy.load(): '${path}' is not a snapshot directory ` +
          `(expected a directory created by db.persist(path))`
      )
    }

    const brain = new Brainy<T>({
      storage: { type: 'filesystem', path },
      mode: 'reader'
    })
    await brain.init()
    return brain.createPinnedDb({
      generation: brain.generationStore.generation(),
      timestamp: Date.now(),
      closeOnRelease: () => brain.close()
    })
  }

  // --- Db host plumbing ------------------------------------------------------

  /**
   * Guard for the synchronous Db entry points (`now()`, `generation()`):
   * they cannot await initialization, so they demand it up front.
   */
  private assertGenerationStoreReady(method: string): void {
    if (!this.initialized || !this.generationStore) {
      throw new Error(
        `Cannot call ${method}() before init() completes. ` +
          `Await brain.init() (or brain.ready, or use Brainy.open()).`
      )
    }
  }

  /**
   * Pin `generation` and construct the `Db` value over the shared host. The
   * `Db` constructor registers itself with the GC-backstop finalization
   * registry; explicit `db.release()` unregisters first.
   */
  private createPinnedDb(init: {
    generation: number
    timestamp: number
    receipt?: TransactReceipt
    closeOnRelease?: () => Promise<void>
  }): Db<T> {
    this.pinGeneration(init.generation)
    return new Db<T>({ host: this.dbHost, ...init })
  }

  /**
   * The host surface every `Db` of this brain shares (lazily built once):
   * live read fast paths, generation-record materialization, snapshot
   * support, and pin lifecycle. See {@link DbHost} in src/db/db.ts.
   */
  private get dbHost(): DbHost<T> {
    if (!this._dbHost) {
      this._dbHost = {
        store: this.generationStore,
        storage: this.storage,
        get: (id, options) => this.get(id, options),
        find: (query) => this.find(query),
        related: (paramsOrId) => this.related(paramsOrId),
        resolveGeneration: (target, options) => this.resolveAsOfGeneration(target, options),
        entityFromRecord: (id, record, includeVectors) =>
          this.entityFromGenerationRecord(id, record, includeVectors),
        relationFromRecord: (id, record) => this.relationFromGenerationRecord(id, record),
        persistPinned: (targetPath, generation) =>
          this.persistPinnedGeneration(targetPath, generation),
        materializeAt: (generation) => this.materializeAtGeneration(generation),
        canServeVectorAtGeneration: (generation) => this.canServeVectorAtGeneration(generation),
        vectorSearchAtGeneration: (params, allowedIds, k, generation) =>
          this.vectorSearchAtGeneration(params, allowedIds, k, generation),
        pinGeneration: (generation) => this.pinGeneration(generation),
        releaseGeneration: (generation) => this.releaseGeneration(generation),
        registerDbForFinalization: (db, generation, closeOnRelease) => {
          this.dbFinalizationRegistry.register(db, { generation, closeOnRelease }, db)
        },
        unregisterDbFromFinalization: (db) => {
          this._dbFinalizationRegistry?.unregister(db)
        }
      }
    }
    return this._dbHost
  }

  /** Lazily build the GC backstop for leaked (never-released) `Db` pins. */
  private get dbFinalizationRegistry(): FinalizationRegistry<{
    generation: number
    closeOnRelease?: () => Promise<void>
  }> {
    if (!this._dbFinalizationRegistry) {
      this._dbFinalizationRegistry = new FinalizationRegistry((held) => {
        this.releaseGeneration(held.generation)
        if (held.closeOnRelease) {
          void held.closeOnRelease().catch((err: Error) => {
            prodLog.warn(`[Brainy] Db finalization close failed: ${err.message}`)
          })
        }
      })
    }
    return this._dbFinalizationRegistry
  }

  /**
   * Registered index providers that implement the optional
   * {@link VersionedIndexProvider} capability (feature-detected on the three
   * index surfaces: vector, metadata, graph). Brainy's own JS indexes do not
   * implement it.
   */
  private versionedIndexProviders(): VersionedIndexProvider[] {
    const candidates: unknown[] = [this.index, this.metadataIndex, this.graphIndex]
    return candidates.filter(isVersionedIndexProvider)
  }

  /**
   * Take one refcounted pin on `generation`: storage-record pin (gates
   * compaction) plus a `pin()` on every versioned index provider — the
   * explicit pin lifetime overrides any time-based snapshot retention the
   * provider has. Provider visibility (`isGenerationVisible`) is evaluated
   * at pin time per the locked read-routing rule. Without a versioned
   * provider, historical reads resolve from canonical generation records
   * and the at-generation index materialization (strictly correct,
   * provider-independent); a provider that retains the pinned generation
   * serves the same reads from its segments without rebuild.
   */
  private pinGeneration(generation: number): void {
    this.generationStore.pin(generation)
    const big = BigInt(generation)
    for (const provider of this.versionedIndexProviders()) {
      const visible = provider.isGenerationVisible(big)
      provider.pin(big)
      if (!visible) {
        prodLog.debug(
          `[Brainy] generation ${generation} pinned but not provider-visible — ` +
            `historical reads at this pin resolve from canonical generation records`
        )
      }
    }
  }

  /** Release one refcounted pin on `generation` (mirror of {@link pinGeneration}). */
  private releaseGeneration(generation: number): void {
    this.generationStore.release(generation)
    const big = BigInt(generation)
    for (const provider of this.versionedIndexProviders()) {
      provider.release(big)
    }
  }

  /**
   * Materialize an `Entity` from a generation record's raw stored objects —
   * the historical-read counterpart of the live `get()` paths.
   * `record.metadata` is the exact stored metadata object;
   * `record.vector` is the stored HNSW noun object (whose `.vector` carries
   * the embedding) or `null` when the vector file was absent.
   */
  private async entityFromGenerationRecord(
    id: string,
    record: { metadata: any; vector: any | null },
    includeVectors: boolean
  ): Promise<Entity<T>> {
    const entity = await this.convertMetadataToEntity(id, record.metadata)
    if (includeVectors && record.vector && Array.isArray(record.vector.vector)) {
      entity.vector = record.vector.vector
    }
    return entity
  }

  /**
   * Materialize a `Relation` from a generation record's raw stored objects.
   * Field split mirrors `storage.getVerb()` + `verbsToRelations()`: the
   * stored vector object carries the structural core (`sourceId`/`targetId`/
   * `verb`), the stored metadata object carries typed fields plus the custom
   * metadata bag. Returns `null` when the metadata part is absent (the
   * relationship did not exist as a live edge).
   * @throws Error when metadata exists but the structural core is missing —
   *   that state is unreachable through the API (relate() writes both parts
   *   in one atomic batch) and indicates external tampering; throwing beats
   *   silently dropping an edge from historical results.
   */
  private relationFromGenerationRecord(
    id: string,
    record: { metadata: any; vector: any | null }
  ): Relation<T> | null {
    if (record.metadata === null || record.metadata === undefined) {
      return null
    }
    const core = record.vector as { sourceId?: string; targetId?: string; verb?: string } | null
    if (!core || typeof core.sourceId !== 'string' || typeof core.targetId !== 'string') {
      throw new Error(
        `Generation record for relationship ${id} has metadata but no structural core ` +
          `(sourceId/targetId) — store corrupted or records modified outside Brainy`
      )
    }

    // Canonical reserved/custom split — same single source of truth as the
    // live verb read paths (see src/types/reservedFields.ts).
    const { reserved, custom } = splitVerbMetadataRecord(
      record.metadata as Record<string, unknown>
    )

    return {
      id,
      from: core.sourceId,
      to: core.targetId,
      type: (reserved.verb ?? core.verb) as VerbType,
      ...(reserved.subtype !== undefined && { subtype: reserved.subtype as string }),
      ...(reserved.visibility !== undefined && { visibility: reserved.visibility as EntityVisibility }),
      weight: (reserved.weight as number) ?? 1.0,
      data: reserved.data,
      metadata: custom as T,
      service: reserved.service as string,
      createdAt: typeof reserved.createdAt === 'number' ? reserved.createdAt : Date.now(),
      ...(typeof reserved.updatedAt === 'number' && { updatedAt: reserved.updatedAt }),
      ...(typeof reserved.confidence === 'number' && { confidence: reserved.confidence })
    }
  }

  /**
   * Snapshot the store at `generation` into `targetPath` — the brain-side
   * half of `db.persist()`. Indexes are flushed first (so the snapshot's
   * persisted index files match its records), then the snapshot is cut
   * under the generation store's commit lock: no transact commit, no
   * compaction, and no counter write can interleave with the hard-link
   * walk, and the counter is durably persisted into the snapshot.
   *
   * @throws GenerationConflictError when `generation` is no longer the
   *   store's latest — a snapshot captures current bytes, so persist the
   *   view before further writes (pin with `brain.now()`, persist, then
   *   mutate).
   */
  private async persistPinnedGeneration(targetPath: string, generation: number): Promise<void> {
    await this.ensureInitialized()
    await this.flush()
    await this.generationStore.snapshotWith(async () => {
      if (generation !== this.generationStore.generation()) {
        throw new GenerationConflictError(generation, this.generationStore.generation())
      }
      await this.storage.snapshotToDirectory(targetPath)
    })
  }

  /**
   * Build the at-`generation` index materialization — the brain-side half of
   * the historical full-query surface (`Db.find`/`Db.related` at
   * past pinned generations; see `src/db/db.ts` and
   * `docs/ADR-001-generational-mvcc.md`):
   *
   * 1. Flush, then enumerate every live entity/relationship id plus every id
   *    touched by transactions committed after `generation`.
   * 2. Resolve each id AT `generation` (live bytes when untouched since the
   *    pin; immutable before-images otherwise) and copy the raw stored
   *    objects into a fresh in-memory storage adapter.
   * 3. A final reconciliation pass under the store's commit mutex
   *    re-resolves ids touched by transactions that committed DURING the
   *    copy, so the materialized set is exactly the at-`generation` record
   *    set. (Single-operation writes racing the copy follow the documented
   *    history granularity — they remain visible through earlier pins.)
   * 4. Open a read-only `Brainy` over that storage: its init reconciliation
   *    rebuilds the metadata and graph-adjacency indexes by scanning the
   *    copied records, and the materializer then builds the
   *    `JsHnswVectorIndex` by inserting every copied at-`generation` vector
   *    (the persisted-HNSW restore path has nothing to restore — the
   *    at-generation graph never existed on disk). The host's embedder is
   *    shared so semantic queries embed through the already-loaded model,
   *    and the host's aggregate definitions are re-registered so
   *    `find({ aggregate })` computes at-generation values via
   *    backfill-on-define.
   *
   * COST — document-grade contract: O(n at G) time and memory, ONCE per
   * `Db` (the `Db` caches the handle; `db.release()` frees it). This is the
   * open-core price of historical index queries. A native
   * `VersionedIndexProvider` serves the same reads from its retained
   * segments without any rebuild — when one is registered, it accelerates
   * these queries instead.
   */
  private async materializeAtGeneration(generation: number): Promise<HistoricalQueryHandle<T>> {
    await this.ensureInitialized()
    await this.flush()

    const snapshotStorage = new MemoryStorage()
    await snapshotStorage.init()

    // Copy one id's at-`generation` state into the snapshot (or ensure absence)
    // as a PURE LOOKUP against a precomputed `firstAfter` map (id → the first
    // generation after the pin that touched it). An id absent from the map was
    // untouched since the pin → its live storage state IS its at-`generation`
    // state; an id present resolves from that generation's immutable
    // before-image. Both `resolveManyAt` (builds the map) and
    // `readGenerationRecord` (reads the before-image) are MUTEX-FREE, so this is
    // safe to call inside the reconciliation pass below (which holds the commit
    // mutex via `snapshotWith`) — a per-id `resolveAt` there would re-enter the
    // mutex and DEADLOCK, and would regress the O(R) bulk copy to O(N·R).
    const copyAt = async (
      kind: 'noun' | 'verb',
      id: string,
      firstAfter: Map<string, number>
    ): Promise<void> => {
      let record: { metadata: any; vector: any | null } | null = null
      const candidate = firstAfter.get(id)
      if (candidate === undefined) {
        // Untouched since the pin → live storage IS the at-`generation` state.
        const raw =
          kind === 'noun' ? await this.storage.readNounRaw(id) : await this.storage.readVerbRaw(id)
        if (raw.metadata !== null) record = raw
      } else {
        const before = await this.generationStore.readGenerationRecord(kind, candidate, id)
        if (before === null) {
          throw new Error(
            `Generation record missing: _generations/${candidate}/prev/${id}.json ` +
              `(store corrupted or records removed outside compactHistory())`
          )
        }
        // A non-null metadata before-image is the at-`generation` live state; a
        // null-metadata before-image (the create sentinel, or a metadata-less
        // stored state) is absent at this generation.
        if (before.metadata !== null) record = { metadata: before.metadata, vector: before.vector }
      }
      // `record === null` — absent at this generation: writing nulls ensures the
      // id is absent in the snapshot, which also un-copies ids created by
      // transactions that commit mid-build.
      if (kind === 'noun') {
        await snapshotStorage.writeNounRaw(id, record ?? { metadata: null, vector: null })
      } else {
        await snapshotStorage.writeVerbRaw(id, record ?? { metadata: null, vector: null })
      }
    }

    // Bulk copy: live ids ∪ ids touched after the pinned generation (the
    // union covers entities deleted after the pin, which are no longer
    // listed live but resolve from before-images).
    let watermark = this.generationStore.committedGeneration()
    const changed = await this.generationStore.changedBetween(generation, watermark)
    const nounIds = new Set<string>(changed.nouns)
    const verbIds = new Set<string>(changed.verbs)
    for (const path of await this.storage.listRawObjects('entities/nouns')) {
      const id = entityIdFromCanonicalPath(path)
      if (id) nounIds.add(id)
    }
    for (const path of await this.storage.listRawObjects('entities/verbs')) {
      const id = entityIdFromCanonicalPath(path)
      if (id) verbIds.add(id)
    }
    // ONE ascending pass per kind resolves the whole universe's first-after
    // generations (O(R) getDelta reads, NOT O(N·R)), then each copy is a lookup.
    const nounFirstAfter = await this.generationStore.resolveManyAt('noun', nounIds, generation)
    const verbFirstAfter = await this.generationStore.resolveManyAt('verb', verbIds, generation)
    for (const id of nounIds) await copyAt('noun', id, nounFirstAfter)
    for (const id of verbIds) await copyAt('verb', id, verbFirstAfter)

    // Reconciliation pass under the commit mutex: nothing can commit during
    // this section, so afterwards the snapshot is exactly the at-`generation`
    // record set even when transactions committed while the bulk copy ran.
    // Reconciled ids join the enumeration sets so the vector-index build
    // below sees them too. `resolveManyAt`/`readGenerationRecord` are mutex-free,
    // so this section never re-enters the commit mutex (no deadlock).
    await this.generationStore.snapshotWith(async () => {
      const committedNow = this.generationStore.committedGeneration()
      if (committedNow === watermark) return
      const delta = await this.generationStore.changedBetween(watermark, committedNow)
      const reconNouns = new Set<string>(delta.nouns)
      const reconVerbs = new Set<string>(delta.verbs)
      const reconNounFirstAfter = await this.generationStore.resolveManyAt(
        'noun',
        reconNouns,
        generation
      )
      const reconVerbFirstAfter = await this.generationStore.resolveManyAt(
        'verb',
        reconVerbs,
        generation
      )
      for (const id of reconNouns) {
        nounIds.add(id)
        await copyAt('noun', id, reconNounFirstAfter)
      }
      for (const id of reconVerbs) {
        verbIds.add(id)
        await copyAt('verb', id, reconVerbFirstAfter)
      }
      watermark = committedNow
    })

    // Open the ephemeral reader: init's index reconciliation rebuilds the
    // metadata and graph indexes by scanning the copied records.
    const reader = new Brainy<T>({
      storage: snapshotStorage,
      mode: 'reader',
      requireSubtype: false,
      silent: true
    })
    await reader.init()
    // Share the host's embedder and dimensionality — semantic queries on the
    // materialization must never load a second embedding model.
    reader.embedder = this.embedder
    reader.dimensions = this.dimensions
    // Build the reader's vector index by INSERTING the copied at-generation
    // vectors. Unlike the metadata/graph indexes (which rebuild by scanning
    // records), `JsHnswVectorIndex.rebuild()` only restores a previously
    // persisted graph structure — and the at-generation HNSW graph never
    // existed on disk, so the materializer constructs it the honest way:
    // one insert per vector (the O(n log n at G) component of the documented
    // materialization cost).
    for (const id of nounIds) {
      const noun = await snapshotStorage.getNoun(id)
      if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) {
        // THE ZERO-NORM LAW: a direct provider-write seam (this materializer
        // inserts one-by-one, bypassing AddToVectorIndexOperation's own
        // belt) — apply the same refusal here rather than handing a false
        // attractor to the ephemeral reader's index.
        if (isZeroNormVector(noun.vector)) {
          prodLog.warn(
            `[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` +
              `entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` +
              `engine boundary (the materialized record is unaffected)`
          )
          continue
        }
        await reader.index.addItem({ id: noun.id, vector: noun.vector })
      }
    }
    // Re-register the host's aggregate definitions; backfill-on-define
    // computes their at-generation values from the materialized record set
    // on first query.
    if (this._aggregationIndex) {
      for (const def of this._aggregationIndex.getDefinitions()) {
        reader.defineAggregate(def)
      }
    }

    let closed = false
    return {
      find: (query) => reader.find(query),
      related: (paramsOrId) => reader.related(paramsOrId),
      close: async () => {
        if (closed) return
        closed = true
        await reader.close()
      }
    }
  }

  /**
   * @description 8.0 #35 — true when the vector index is a
   * {@link VersionedIndexProvider} that advertises `isGenerationVisible(generation)`:
   * it can serve the at-`generation` vector leg from retained segments, so a
   * historical filtered semantic read needs no O(n@G) JS-HNSW rebuild. The built-in
   * `JsHnswVectorIndex` is not versioned (only ever "now"), and a native provider
   * that cannot honor `generation` MUST return `false` here (refuse, never fabricate
   * now-vectors-as-at-gen), so the caller falls back to materialization.
   */
  private canServeVectorAtGeneration(generation: number): boolean {
    return (
      isVersionedIndexProvider(this.index) &&
      this.index.isGenerationVisible(BigInt(generation))
    )
  }

  /**
   * @description 8.0 #35 — run the vector kNN AS OF `generation`, restricted to
   * `allowedIds` (the at-gen metadata∩graph universe the `Db` resolved from the
   * record layer). The versioned provider serves the at-gen walk; Brainy composes
   * the metadata half. Only reached when {@link canServeVectorAtGeneration} is true.
   * @param params - The semantic (`query`) or explicit-`vector` find params.
   * @param allowedIds - The at-gen candidate universe (membership-correct at `generation`).
   * @param k - Over-fetch (page + headroom).
   * @param generation - The as-of generation (Brainy's u64 commit counter).
   * @returns `[id, distance]` pairs, ascending distance (descending relevance).
   */
  private async vectorSearchAtGeneration(
    params: FindParams<T>,
    allowedIds: ReadonlySet<string>,
    k: number,
    generation: number
  ): Promise<Array<[string, number]>> {
    const vector = params.vector || (await this.embed(params.query!))
    // #35 part-3: supply the at-gen candidate VECTORS (resolved from Brainy's
    // per-generation record before-images) so the provider reranks the historically
    // correct vectors with zero per-vector crossing. `atGenerationVectors.ids` is the
    // candidate set; `allowedIds` rides along (the provider may AND it).
    const atGenerationVectors = await this.buildAtGenerationVectors(allowedIds, generation)
    return this.index.search(vector, k, undefined, {
      allowedIds,
      generation: BigInt(generation),
      ...(atGenerationVectors && { atGenerationVectors })
    })
  }

  /**
   * @description 8.0 #35 part-3 — assemble the {@link AtGenerationVectors} columnar
   * payload for a filtered at-gen exact-rerank: resolve each universe id's vector AS
   * OF `generation` (the canonical record before-image, or live storage when the id
   * was untouched since the pin), intern the id via the shared mapper, and pack the
   * vectors flat row-major (`vectors[i*dim .. (i+1)*dim]` ↔ `ids[i]`). Ids absent at
   * the generation, vectorless, or of the wrong dimension are dropped (they cannot be
   * vector-ranked). Bounded by the filtered universe — far cheaper than the full-corpus
   * rebuild it replaces. Returns `undefined` when no dimension is known or nothing maps.
   * @param allowedIds - The at-gen filtered universe (candidate ids).
   * @param generation - The as-of generation.
   * @returns The columnar payload, or `undefined` when there is nothing to ship.
   */
  private async buildAtGenerationVectors(
    allowedIds: ReadonlySet<string>,
    generation: number
  ): Promise<AtGenerationVectors | undefined> {
    const dim = this.dimensions
    if (!dim || allowedIds.size === 0) return undefined

    const idMapper = this.metadataIndex.getIdMapper()
    const ints: bigint[] = []
    const rows: number[][] = []
    for (const id of allowedIds) {
      const resolved = await this.generationStore.resolveAt('noun', id, generation)
      let vec: number[] | null = null
      if (resolved.source === 'record') {
        // The generation record carries the at-gen vector before-image.
        vec = resolved.vector
      } else if (resolved.source === 'current') {
        // Untouched since the pin → the live vector IS the at-gen vector. Use
        // getNoun (full hydration, lazy-load aware) — readNounRaw's canonical path
        // is empty under lazy-vector eviction.
        const noun = await this.storage.getNoun(id)
        vec = noun?.vector ?? null
      }
      // 'absent' / vectorless / wrong-dim → skip (not vector-rankable at this gen).
      if (Array.isArray(vec) && vec.length === dim) {
        // Mint-now fallback stamps the CURRENT committed watermark (the mint
        // happens now, regardless of the historical G being materialized).
        ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id, this.indexWriteGeneration())))
        rows.push(vec)
      }
    }
    if (ints.length === 0) return undefined

    // Row-major pack: INVARIANT vectors.length === ids.length * dim.
    const vectors = new Float32Array(ints.length * dim)
    for (let i = 0; i < rows.length; i++) vectors.set(rows[i], i * dim)
    return { ids: BigInt64Array.from(ints), vectors, dim }
  }

  // --- Transact planner ------------------------------------------------------

  /**
   * Plan a full `transact()` batch: validate and resolve every operation
   * against "current state + batch so far", producing the ordered
   * TransactionManager operations, the receipt ids, the touched-id sets
   * (which the generation store stages before-images for), and the
   * post-commit aggregation hooks. Planning performs reads and embedding
   * only — nothing touches storage until the generation store runs the
   * batch.
   */
  private async planTransact(ops: TxOperation<T>[]): Promise<PlannedTransact> {
    const state: TxPlanState = {
      nouns: new Map(),
      removedNouns: new Set(),
      verbs: new Map(),
      removedVerbs: new Set()
    }
    const plan: PlannedTransact = {
      operations: [],
      ids: [],
      touchedNouns: [],
      touchedVerbs: [],
      postCommit: [],
      casUpdates: [],
      createdNouns: new Set(),
      changeEvents: [],
      markerRecords: [],
      vectorUnlands: []
    }

    for (const op of ops) {
      switch (op.op) {
        case 'add':
          plan.ids.push(await this.planTxAdd(op, state, plan))
          break
        case 'update':
          plan.ids.push(await this.planTxUpdate(op, state, plan))
          break
        case 'remove':
          plan.ids.push(await this.planTxRemove(op, state, plan))
          break
        case 'relate':
          plan.ids.push(await this.planTxRelate(op, state, plan))
          break
        case 'unrelate':
          plan.ids.push(await this.planTxUnrelate(op, state, plan))
          break
        default: {
          const exhaustive: never = op
          throw new Error(`transact(): unknown operation ${JSON.stringify(exhaustive)}`)
        }
      }
    }

    return plan
  }

  /**
   * Resolve an entity during planning: batch-pending writes win, then the
   * live store. `includeVectors: false` returns the stub vector exactly as
   * the live metadata-only `get()` does, so planned relationship vectors
   * match `relate()` byte-for-byte.
   */
  private async planGetEntity(
    state: TxPlanState,
    id: string,
    options?: GetOptions
  ): Promise<Entity<T> | null> {
    if (state.removedNouns.has(id)) {
      return null
    }
    const pending = state.nouns.get(id)
    if (pending) {
      const entity = await this.convertMetadataToEntity(id, pending.metadata)
      if (options?.includeVectors) {
        entity.vector = pending.vector
      }
      return entity
    }
    return this.get(id, options)
  }

  /** Plan one `{ op: 'add' }` — mirror of `add()`. Returns the entity id. */
  private async planTxAdd(
    op: Extract<TxOperation<T>, { op: 'add' }>,
    state: TxPlanState,
    plan: PlannedTransact
  ): Promise<string> {
    const { op: _discriminator, ...rawParams } = op
    validateAddParams(rawParams as AddParams<T>)
    const params = rawParams as AddParams<T>
    this.enforceTrackedFieldValues(params.metadata as Record<string, unknown> | undefined, 'metadata')
    this.enforceTrackedFieldValues({ subtype: params.subtype } as Record<string, unknown>, 'top-level')
    this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata)

    // Id normalization (8.0) — mirror of add(): a natural key coerces to a
    // STABLE UUID (v5), preserving the original under ORIGINAL_ID_KEY; a real
    // UUID passes through; no id mints a fresh v7. Done BEFORE ifAbsent so a
    // natural-key upsert is idempotent against the same key.
    const { id, originalId } = coerceNewEntityId(params.id)

    // ifAbsent and upsert resolve the same id collision in opposite ways
    // (skip vs. merge) — mirror of add()'s hard error.
    if (params.ifAbsent && params.upsert) {
      throw new Error(
        'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
      )
    }

    // ifAbsent — idempotent by-id insert, resolved against batch + store.
    if (params.id && params.ifAbsent) {
      const existing = await this.planGetEntity(state, id)
      if (existing) {
        return id
      }
    }

    // upsert — by-id create-or-update, resolved against batch + store. When the id
    // already exists, delegate to planTxUpdate with a synthesized update op so the
    // supplied fields MERGE (mirror of add()'s live upsert path) instead of
    // overwriting. The id is already canonical here (coerceNewEntityId above).
    if (params.id && params.upsert) {
      const existing = await this.planGetEntity(state, id)
      if (existing) {
        return this.planTxUpdate(
          {
            op: 'update',
            id,
            ...(params.data !== undefined && { data: params.data }),
            ...(params.type !== undefined && { type: params.type }),
            ...(params.subtype !== undefined && { subtype: params.subtype }),
            ...(params.visibility !== undefined && { visibility: params.visibility }),
            ...(params.metadata !== undefined && { metadata: params.metadata }),
            ...(params.vector !== undefined && { vector: params.vector }),
            ...(params.confidence !== undefined && { confidence: params.confidence }),
            ...(params.weight !== undefined && { weight: params.weight })
          } as Extract<TxOperation<T>, { op: 'update' }>,
          state,
          plan
        )
      }
    }

    // MT5 deferred embedding: ack at durability with a stub vector and a
    // DURABLE pending marker (written BEFORE the commit — an orphaned marker
    // from a failed commit is harmless and reaped by the worker; a
    // marker-less committed row would be a silently missing vector, which is
    // the disallowed direction). The background worker embeds + inserts.
    const deferringEmbed = params.deferEmbedding === true && !params.vector
    let vector = deferringEmbed
      ? []
      : params.vector || (await this.embed(params.data))

    // THE ZERO-NORM LAW — see the single-add() insert path's matching
    // comment (a zero-norm vector is not a vector; never crosses an engine
    // boundary). Normalized here BEFORE the dimension pin and the
    // vectored-ledger `hasVector` flag below ever see it.
    if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) {
      prodLog.warn(
        `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` +
          `a zero-norm vector is not a vector; persisted unvectored ([]) instead.`
      )
      vector = []
    }

    // Gated on `vector.length > 0` — see the single-add() insert path's
    // matching comment: an explicit `vector: []` carries no dimension
    // information either, deferred or not.
    if (!deferringEmbed && vector.length > 0) {
      if (!this.dimensions) {
        this.dimensions = vector.length
      } else if (vector.length !== this.dimensions) {
        throw new Error(
          `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}`
        )
      }
    }

    // isNew controls the operation's rollback strategy: a custom id may
    // collide with an existing entity (add() overwrite semantics), and a
    // failed batch must restore the overwritten bytes.
    const isNew = params.id
      ? !state.nouns.has(id) && (await this.storage.getNounMetadata(id)) === null
      : true

    // Either way (fresh create OR overwrite) this add resets the id's revision
    // baseline (`_rev: 1` below), so later in-batch updates to it sequence
    // against batch-local state — see PlannedTransact.createdNouns.
    plan.createdNouns.add(id)

    const now = Date.now()
    // v2 nested-bag record — mirror of add(): engine fields top-level, the
    // user's bag nested verbatim (collider names stay the user's).
    const storageMetadata = buildNounMetadataRecord(
      {
        data: params.data,
        noun: params.type,
        ...(params.subtype !== undefined && { subtype: params.subtype }),
        // visibility: stored only when not 'public' (absent === public, keeps records lean)
        ...(params.visibility !== undefined &&
          params.visibility !== 'public' && { visibility: params.visibility }),
        service: params.service,
        createdAt: now,
        updatedAt: now,
        _rev: 1,
        ...(params.confidence !== undefined && { confidence: params.confidence }),
        ...(params.weight !== undefined && { weight: params.weight }),
        ...(params.createdBy && { createdBy: params.createdBy })
      },
      {
        ...params.metadata,
        // Preserve the caller's original (non-UUID) id when normalized — mirror
        // of add(). A real UUID passes through with no _originalId.
        ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId })
      }
    )
    const entityForIndexing = {
      id,
      vector,
      connections: new Map(),
      // no `level` — plumbing never enters the indexing view
      type: params.type,
      ...(params.subtype !== undefined && { subtype: params.subtype }),
      ...(params.visibility !== undefined &&
        params.visibility !== 'public' && { visibility: params.visibility }),
      ...(params.confidence !== undefined && { confidence: params.confidence }),
      ...(params.weight !== undefined && { weight: params.weight }),
      createdAt: now,
      updatedAt: now,
      service: params.service,
      data: params.data,
      ...(params.createdBy && { createdBy: params.createdBy }),
      metadata: {
        ...(params.metadata || {}),
        ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId })
      }
    }

    if (deferringEmbed) {
      // The pending marker rides the batch's ONE commit fact (same
      // generation, one atomic append); the worker kicks post-commit via
      // the plan hook.
      plan.markerRecords.push(this.enqueuePendingEmbed(id))
      plan.postCommit.push(() => this.kickEmbedWorker())
    }
    plan.operations.push(
      // hasVector: see the single-add() insert path's comment — never true
      // for a deferred embed (stub vector `[]`; counted later at landing).
      new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
      new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
      // Gated on `vector.length > 0` — see the single-add() insert path's
      // matching comment: an explicit `vector: []` has nothing to index
      // either, deferred or not.
      ...(vector.length > 0
        ? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]
        : []),
      new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration)
    )
    plan.touchedNouns.push(id)
    plan.postCommit.push(() => {
      if (this._aggregationIndex) {
        this._aggregationIndex.onEntityAdded(id, entityForIndexing)
      }
    })
    if (this._changeFeed.hasListeners) {
      plan.changeEvents.push({
        kind: 'entity',
        op: 'add',
        id,
        entity: {
          id,
          type: String(params.type),
          ...(params.subtype !== undefined && { subtype: String(params.subtype) }),
          metadata: (params.metadata as Record<string, unknown>) ?? {},
          ...(params.service !== undefined && { service: String(params.service) })
        }
      })
    }

    state.nouns.set(id, { metadata: storageMetadata, vector })
    state.removedNouns.delete(id)
    return id
  }

  /** Plan one `{ op: 'update' }` — mirror of `update()`. Returns the entity id. */
  private async planTxUpdate(
    op: Extract<TxOperation<T>, { op: 'update' }>,
    state: TxPlanState,
    plan: PlannedTransact
  ): Promise<string> {
    const { op: _discriminator, ...rawParams } = op
    validateUpdateParams(rawParams as UpdateParams<T>)
    const params = rawParams as UpdateParams<T>
    // Id normalization (8.0) — mirror of update(): a natural key resolves to the
    // canonical UUID add() stored. A real UUID passes through.
    params.id = resolveEntityId(params.id)
    this.enforceTrackedFieldValues(params.metadata as Record<string, unknown> | undefined, 'metadata')
    if (params.subtype !== undefined) {
      this.enforceTrackedFieldValues({ subtype: params.subtype } as Record<string, unknown>, 'top-level')
    }

    const existing = await this.planGetEntity(state, params.id, { includeVectors: true })
    if (!existing) {
      throw new EntityNotFoundError(params.id)
    }

    if (params.subtype !== undefined || params.type !== undefined) {
      const effectiveType = params.type ?? existing.type
      const effectiveSubtype = params.subtype !== undefined ? params.subtype : existing.subtype
      const effectiveMetadata = params.metadata ?? existing.metadata
      this.enforceSubtypeOnAdd('update', effectiveType, effectiveSubtype, effectiveMetadata)
    }

    // ifRev CAS — identical resolution to update(); a conflict rejects the
    // WHOLE batch. This planning-time check is purely a FAST-FAIL (it can
    // interleave with concurrent writers); the authoritative re-verify runs in
    // transact()'s commit precondition under the commit mutex, via the
    // plan.casUpdates entry registered below.
    const currentRev = typeof existing._rev === 'number' ? existing._rev : 1
    if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) {
      throw new RevisionConflictError(params.id, params.ifRev, currentRev)
    }

    // Resolve the updated vector — mirror of update(): an explicit `vector`
    // always wins, new `data` re-embeds, otherwise the existing vector is
    // kept. Any vector change re-indexes HNSW below.
    // 'data' is present whenever it's not null/undefined — '' is real
    // content (see the identical hasNewData in update()); a plain truthy
    // check would silently skip re-embedding an emptied value and leave a
    // stale vector with no path to ever correct itself.
    const rawHasNewData = params.data !== undefined && params.data !== null
    // No re-embed on unchanged data — the transact() mirror of update()'s rule.
    const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data)
    const hasNewData = rawHasNewData && !dataUnchanged
    let vector = existing.vector

    // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror
    // of update()'s matching block: an explicit REAL all-zero vector
    // normalizes to `[]` (never crosses an engine boundary), and an
    // explicit `vector: []` (post-normalization) is the sanctioned unvector
    // instruction, exempt from the dimension check. `validateUpdateParams`
    // already refuses combining it with `deferEmbedding: true`.
    let explicitVector = params.vector
    if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) {
      prodLog.warn(
        `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` +
          `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.`
      )
      explicitVector = []
    }
    const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0

    if (explicitVector) {
      if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) {
        throw new Error(
          `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}`
        )
      }
      vector = explicitVector
    } else if (hasNewData) {
      vector = await this.embed(params.data)
    }
    const needsReindexing = Boolean(hasNewData || params.type || explicitVector)

    // Leg D — the unvector door clears a PENDING deferred-embed marker (see
    // update()'s matching comment for the full rationale): the durable
    // clear (an `embed.landed` record, empty vector) rides the batch's ONE
    // commit fact via `plan.markerRecords`; the in-memory clear is deferred
    // to `plan.postCommit` so an aborted batch never disarms a marker whose
    // durable twin was never written.
    const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id)
    if (clearsPendingEmbed) {
      plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] })
      plan.postCommit.push(() => {
        this.clearPendingEmbed(params.id)
        prodLog.warn(
          `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` +
            `the unvector door cleared it ('vector: []' is an explicit instruction, ` +
            `never "please embed").`
        )
      })
    }

    // Leg D — vectored-ledger decrement for the sanctioned unvector door,
    // deferred to `plan.vectorUnlands` (consumed with a proper `await` in
    // `transact()`, AFTER the commit succeeds — see its matching comment).
    // Gated on the PRIOR vector having actually been real (non-empty,
    // non-zero-norm): idempotent on an already-unvectored row.
    if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) {
      plan.vectorUnlands.push(params.id)
    }

    const newMetadata =
      params.merge !== false
        ? { ...existing.metadata, ...params.metadata }
        : params.metadata || existing.metadata
    const now = Date.now()
    // v2 nested-bag record — mirror of update(): engine fields top-level,
    // the merged user bag nested verbatim.
    const updatedMetadata = buildNounMetadataRecord(
      {
        data: params.data !== undefined ? params.data : existing.data,
        noun: params.type || existing.type,
        service: existing.service,
        createdAt: existing.createdAt,
        updatedAt: now,
        _rev: currentRev + 1,
        ...(params.confidence !== undefined && { confidence: params.confidence }),
        ...(params.weight !== undefined && { weight: params.weight }),
        ...(params.confidence === undefined &&
          existing.confidence !== undefined && { confidence: existing.confidence }),
        ...(params.weight === undefined &&
          existing.weight !== undefined && { weight: existing.weight }),
        ...(params.subtype !== undefined && { subtype: params.subtype }),
        ...(params.subtype === undefined &&
          existing.subtype !== undefined && { subtype: existing.subtype }),
        // Visibility: new value if provided, else preserve existing; stored only when the
        // effective value is not 'public' (a change to 'public' drops the field).
        ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
          visibility: params.visibility ?? existing.visibility
        })
      },
      newMetadata as Record<string, unknown>
    )

    // Register for the authoritative under-mutex CAS re-verify + rev re-stamp
    // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation
    // holds `updatedMetadata` by reference, so the precommit re-stamp lands in
    // what execute() writes.
    plan.casUpdates.push({
      id: params.id,
      ...(typeof params.ifRev === 'number' && { ifRev: params.ifRev }),
      updatedMetadata
    })

    const entityForIndexing = {
      id: params.id,
      vector,
      connections: new Map(),
      // no `level` — plumbing never enters the indexing view
      type: params.type || existing.type,
      subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
      ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
        visibility: params.visibility ?? existing.visibility
      }),
      confidence: params.confidence !== undefined ? params.confidence : existing.confidence,
      weight: params.weight !== undefined ? params.weight : existing.weight,
      createdAt: existing.createdAt,
      updatedAt: now,
      service: existing.service,
      data: params.data !== undefined ? params.data : existing.data,
      createdBy: existing.createdBy,
      metadata: newMetadata
    }
    const removalMetadata = {
      type: existing.type,
      confidence: existing.confidence,
      weight: existing.weight,
      createdAt: existing.createdAt,
      updatedAt: existing.updatedAt,
      service: existing.service,
      data: existing.data,
      createdBy: existing.createdBy,
      metadata: existing.metadata
    }

    plan.operations.push(
      new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
    )
    // Noun-record write + HNSW reindex ONLY when the vector side actually
    // changed — the same write-granularity law as update(): a metadata-only
    // patch must never rewrite the whole vector record. This plan path is the
    // one transact() updates ride, so an unconditional save here would
    // re-open the read-sweep disk-saturation amplifier for exactly the
    // consumers batching their stat touches through transact().
    if (needsReindexing) {
      plan.operations.push(
        new SaveNounOperation(this.storage, {
          id: params.id,
          vector,
          connections: new Map(),
          level: 0
        }),
        // ONE atomic vector-index leg — same law as update(): the row must
        // never be absent from vector search during an update (see
        // ReplaceInVectorIndexOperation).
        new ReplaceInVectorIndexOperation(this.index, params.id, existing.vector, vector, this.indexWriteGeneration)
      )
    }
    plan.operations.push(
      new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration),
      new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration)
    )
    plan.touchedNouns.push(params.id)

    // The full planGetEntity view, passed whole — a subset view makes the
    // old-side decrement miss reserved-field groups (double-count on update).
    const oldEntityForAgg = existing as unknown as Record<string, unknown>
    plan.postCommit.push(() => {
      if (this._aggregationIndex) {
        this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg)
      }
    })
    if (this._changeFeed.hasListeners) {
      plan.changeEvents.push({
        kind: 'entity',
        op: 'update',
        id: params.id,
        entity: {
          id: params.id,
          type: String(entityForIndexing.type),
          ...(entityForIndexing.subtype !== undefined && {
            subtype: String(entityForIndexing.subtype)
          }),
          metadata: (newMetadata as Record<string, unknown>) ?? {},
          ...(entityForIndexing.service !== undefined && {
            service: String(entityForIndexing.service)
          })
        }
      })
    }

    state.nouns.set(params.id, { metadata: updatedMetadata, vector })
    return params.id
  }

  /**
   * Plan one `{ op: 'remove' }` — mirror of `remove()`, including the
   * relationship cascade (every edge where the entity is source or target,
   * plus edges created earlier in this batch). Returns the entity id.
   */
  private async planTxRemove(
    op: Extract<TxOperation<T>, { op: 'remove' }>,
    state: TxPlanState,
    plan: PlannedTransact
  ): Promise<string> {
    if (!op.id || typeof op.id !== 'string') {
      throw new Error(`transact(): remove operation requires an entity id (got ${JSON.stringify(op.id)})`)
    }
    // Id normalization (8.0) — mirror of remove(): a natural key resolves to the
    // canonical UUID add() stored. A real UUID passes through.
    const id = resolveEntityId(op.id)

    const pending = state.nouns.get(id)
    const metadata = pending ? pending.metadata : await this.storage.getNounMetadata(id)
    const noun = pending
      ? { id, vector: pending.vector, connections: new Map(), level: 0 }
      : await this.storage.getNoun(id)

    // Cascade set: stored edges touching the entity, plus batch-pending
    // edges, minus edges already removed in this batch.
    const storedVerbs = [
      ...(await this.storage.getVerbsBySource(id)),
      ...(await this.storage.getVerbsByTarget(id))
    ]
    const cascade = new Map<string, GraphVerb>()
    for (const verb of storedVerbs) {
      if (!state.removedVerbs.has(verb.id)) {
        cascade.set(verb.id, verb)
      }
    }
    for (const [verbId, verb] of state.verbs) {
      if (
        !state.removedVerbs.has(verbId) &&
        (verb.sourceId === id || verb.targetId === id)
      ) {
        cascade.set(verbId, verb)
      }
    }

    if (noun) {
      plan.operations.push(new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration))
    }
    if (metadata) {
      plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration))
    }
    // Pre-read metadata rides along: the count decrement must not depend on
    // re-reading the record being removed (see remove()).
    plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata))
    for (const verb of cascade.values()) {
      plan.operations.push(
        // Endpoint ints resolve at EXECUTE time — a cascade verb (or its
        // endpoints) may have been created earlier in this same batch, so a
        // plan-time resolution would ask the id mapper about entities that do
        // not exist yet (the native mapper rightly refuses).
        new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration),
        new DeleteVerbMetadataOperation(this.storage, verb.id)
      )
      // Retract the cascaded relation's metadata-index row too — the
      // transact() mirror of remove()'s single-op cascade leg
      // (null-metadata-safe; see metadataIndexRetractionOp's JSDoc).
      {
        const cascadeRetractionOp = this.metadataIndexRetractionOp(
          verb.id, verb, `transact remove(${id}) cascade unrelate ${verb.id}`
        )
        if (cascadeRetractionOp) plan.operations.push(cascadeRetractionOp)
      }
      plan.touchedVerbs.push(verb.id)
      state.verbs.delete(verb.id)
      state.removedVerbs.add(verb.id)
      if (this._changeFeed.hasListeners) {
        plan.changeEvents.push({
          kind: 'relation',
          op: 'unrelate',
          id: verb.id,
          relation: {
            id: verb.id,
            from: verb.sourceId,
            to: verb.targetId,
            type: String(verb.verb)
          }
        })
      }
    }
    plan.touchedNouns.push(id)
    if (this._changeFeed.hasListeners) {
      plan.changeEvents.push({
        kind: 'entity',
        op: 'remove',
        id,
        ...(metadata && {
          entity: this.entityViewFromRawRecord(id, metadata as Record<string, unknown>)
        })
      })
    }

    if (metadata) {
      // Mirror of remove()'s aggregation hook — the FULL reserved view, so
      // reserved-field groupBy decrements find their group.
      const entityForAgg = this.entityForAggFromRawRecord(metadata as Record<string, unknown>)
      plan.postCommit.push(() => {
        if (this._aggregationIndex) {
          this._aggregationIndex.onEntityDeleted(id, entityForAgg)
        }
      })
    } else {
      // Un-gated (mirror of remove()): a before-image-less delete flags an
      // exact rescan instead of silently skipping the decrement.
      plan.postCommit.push(() => {
        this._aggregationIndex?.flagAllForRescan(
          `transact delete of ${id} carried no before-image metadata — contribution unknowable`
        )
      })
    }

    state.nouns.delete(id)
    state.removedNouns.add(id)
    return id
  }

  /**
   * Plan one `{ op: 'relate' }` — mirror of `relate()`, including duplicate
   * deduplication (against the store AND earlier batch operations) and
   * `bidirectional`. Returns the (primary) relationship id — the existing id
   * when deduplicated.
   */
  private async planTxRelate(
    op: Extract<TxOperation<T>, { op: 'relate' }>,
    state: TxPlanState,
    plan: PlannedTransact
  ): Promise<string> {
    const { op: _discriminator, ...rawParams } = op
    validateRelateParams(rawParams as RelateParams<T>)
    const params = rawParams as RelateParams<T>
    // Id normalization (8.0) — mirror of relate(): resolve BOTH endpoints to the
    // canonical UUID add() stored, so a relate op may reference either side by
    // natural key. Real UUIDs pass through. (Relationship ids are engine-minted.)
    params.from = resolveEntityId(params.from)
    params.to = resolveEntityId(params.to)
    this.enforceSubtypeOnRelate('relate', params.type, params.subtype, params.metadata)

    const fromEntity = await this.planGetEntity(state, params.from)
    const toEntity = await this.planGetEntity(state, params.to)
    if (!fromEntity) {
      throw new EntityNotFoundError(params.from, `Source entity ${params.from} not found`)
    }
    if (!toEntity) {
      throw new EntityNotFoundError(params.to, `Target entity ${params.to} not found`)
    }

    // Dedupe against earlier operations of this batch first…
    for (const [verbId, verb] of state.verbs) {
      if (
        !state.removedVerbs.has(verbId) &&
        verb.sourceId === params.from &&
        verb.targetId === params.to &&
        verb.verb === params.type
      ) {
        return verbId
      }
    }
    // …then against the committed graph (same lookup relate() performs).
    const dupSourceInt = this.graphEntityInt(params.from)
    const verbIds =
      dupSourceInt === undefined
        ? []
        : await this.resolveVerbIntsToIds(await this.graphIndex.getVerbIdsBySource(dupSourceInt))
    if (verbIds.length > 0) {
      const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds)
      for (const verb of verbsMap.values()) {
        if (
          !state.removedVerbs.has(verb.id) &&
          verb.targetId === params.to &&
          verb.verb === params.type
        ) {
          return verb.id
        }
      }
    }

    const id = uuidv4()
    const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2)
    const now = Date.now()
    // v2 nested-bag record — mirror of relate(): engine fields top-level,
    // the user's edge bag nested verbatim.
    const verbMetadata = buildVerbMetadataRecord(
      {
        verb: params.type,
        ...(params.subtype !== undefined && { subtype: params.subtype }),
        // visibility: stored only when not 'public' (absent === public, keeps records lean)
        ...(params.visibility !== undefined &&
          params.visibility !== 'public' && { visibility: params.visibility }),
        weight: params.weight ?? 1.0,
        ...(params.confidence !== undefined && { confidence: params.confidence }),
        ...(params.service !== undefined && { service: params.service }),
        createdAt: now,
        ...(params.data !== undefined && { data: params.data })
      },
      (params.metadata as Record<string, unknown>) || {}
    )
    const verb: GraphVerb = {
      id,
      vector: relationVector,
      sourceId: params.from,
      targetId: params.to,
      verb: params.type,
      type: params.type,
      ...(params.subtype !== undefined && { subtype: params.subtype }),
      ...(params.visibility !== undefined &&
        params.visibility !== 'public' && { visibility: params.visibility }),
      weight: params.weight ?? 1.0,
      ...(params.confidence !== undefined && { confidence: params.confidence }),
      ...(params.service !== undefined && { service: params.service }),
      metadata: params.metadata,
      data: params.data,
      createdAt: now
    }
    plan.operations.push(
      new SaveVerbOperation(this.storage, {
        id,
        vector: relationVector,
        connections: new Map(),
        verb: params.type,
        sourceId: params.from,
        targetId: params.to
      }),
      new SaveVerbMetadataOperation(this.storage, id, verbMetadata),
      // Endpoint ints resolve at EXECUTE time — after any same-batch add of
      // an endpoint has applied. Plan-time resolution was a consumer-reported
      // native/JS parity bug: transact([add X, relate →X]) asked the native
      // id mapper to assign an int for an entity that did not exist yet.
      new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) =>
        this.cacheVerbInt(verbInt, id)
      ),
      // The transact() mirror of relate()'s metadata-index leg — same
      // commit as the graph leg, same raw stored shape.
      new AddToMetadataIndexOperation(this.metadataIndex, id, verbMetadata, this.indexWriteGeneration)
    )
    plan.touchedVerbs.push(id)
    state.verbs.set(id, verb)
    state.removedVerbs.delete(id)
    if (this._changeFeed.hasListeners) {
      plan.changeEvents.push({
        kind: 'relation',
        op: 'relate',
        id,
        relation: {
          id,
          from: params.from,
          to: params.to,
          type: String(params.type),
          ...(params.metadata && { metadata: params.metadata as Record<string, unknown> })
        }
      })
    }

    if (params.bidirectional) {
      const reverseId = uuidv4()
      const reverseVerb: GraphVerb = {
        ...verb,
        id: reverseId,
        sourceId: params.to,
        targetId: params.from
        // sourceInt/targetInt are mirrored onto this object when the graph
        // operation resolves endpoints at execute time.
      }
      plan.operations.push(
        new SaveVerbOperation(this.storage, {
          id: reverseId,
          vector: relationVector,
          connections: new Map(),
          verb: params.type,
          sourceId: params.to,
          targetId: params.from
        }),
        new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata),
        new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) =>
          this.cacheVerbInt(verbInt, reverseId)
        ),
        new AddToMetadataIndexOperation(this.metadataIndex, reverseId, verbMetadata, this.indexWriteGeneration)
      )
      plan.touchedVerbs.push(reverseId)
      state.verbs.set(reverseId, reverseVerb)
      state.removedVerbs.delete(reverseId)
      if (this._changeFeed.hasListeners) {
        plan.changeEvents.push({
          kind: 'relation',
          op: 'relate',
          id: reverseId,
          relation: {
            id: reverseId,
            from: params.to,
            to: params.from,
            type: String(params.type),
            ...(params.metadata && { metadata: params.metadata as Record<string, unknown> })
          }
        })
      }
    }

    return id
  }

  /** Plan one `{ op: 'unrelate' }` — mirror of `unrelate()`. Returns the relationship id. */
  private async planTxUnrelate(
    op: Extract<TxOperation<T>, { op: 'unrelate' }>,
    state: TxPlanState,
    plan: PlannedTransact
  ): Promise<string> {
    if (!op.id || typeof op.id !== 'string') {
      throw new Error(`transact(): unrelate operation requires a relationship id (got ${JSON.stringify(op.id)})`)
    }
    const id = op.id

    const verb = state.removedVerbs.has(id)
      ? null
      : (state.verbs.get(id) ?? (await this.storage.getVerb(id)))

    if (verb) {
      plan.operations.push(
        // Endpoint ints resolve at EXECUTE time — the verb (or its endpoints)
        // may have been created earlier in this same batch (forward refs).
        new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration)
      )
      // The transact() mirror of unrelate()'s metadata-index leg
      // (null-metadata-safe; see metadataIndexRetractionOp's JSDoc — a
      // present `verb` here is never metadata-omitted, but the closure
      // stays defensive rather than assuming).
      const retractionOp = this.metadataIndexRetractionOp(id, verb, `transact unrelate(${id})`)
      if (retractionOp) plan.operations.push(retractionOp)
    }
    plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id))
    plan.touchedVerbs.push(id)
    if (this._changeFeed.hasListeners) {
      plan.changeEvents.push({
        kind: 'relation',
        op: 'unrelate',
        id,
        ...(verb && {
          relation: {
            id,
            from: verb.sourceId,
            to: verb.targetId,
            type: String(verb.verb)
          }
        })
      })
    }

    state.verbs.delete(id)
    state.removedVerbs.add(id)
    return id
  }

  /**
   * Get total count of nouns - O(1) operation
   * @returns Promise that resolves to the total number of nouns
   */
  async getNounCount(): Promise<number> {
    await this.ensureInitialized()
    return this.storage.getNounCount()
  }

  /**
   * Get total count of verbs - O(1) operation
   * @returns Promise that resolves to the total number of verbs
   */
  async getVerbCount(): Promise<number> {
    await this.ensureInitialized()
    return this.storage.getVerbCount()
  }

  /**
   * Get memory statistics and limits
   *
   * Returns detailed memory information including:
   * - Current heap usage
   * - Container memory limits (if detected)
   * - Query limits and how they were calculated
   * - Memory allocation recommendations
   *
   * Use this to debug why query limits are low or to understand
   * memory allocation in production environments.
   *
   * @returns Memory statistics and configuration
   *
   * @example
   * ```typescript
   * const stats = brain.getMemoryStats()
   * console.log(`Query limit: ${stats.limits.maxQueryLimit}`)
   * console.log(`Basis: ${stats.limits.basis}`)
   * console.log(`Free memory: ${Math.round(stats.memory.free / 1024 / 1024)}MB`)
   * ```
   */
  getMemoryStats(): {
    memory: {
      heapUsed: number
      heapTotal: number
      external: number
      rss: number
      free: number
      total: number
      containerLimit: number | null
    }
    limits: {
      maxQueryLimit: number
      maxQueryLength: number
      maxVectorDimensions: number
      basis: 'override' | 'reservedMemory' | 'containerMemory' | 'freeMemory'
    }
    config: {
      maxQueryLimit?: number
      reservedQueryMemory?: number
    }
    recommendations?: string[]
  } {
    const config = ValidationConfig.getInstance()
    const heapStats = process.memoryUsage ? process.memoryUsage() : {
      heapUsed: 0,
      heapTotal: 0,
      external: 0,
      rss: 0
    }

    // Get system memory info
    let freeMemory = 0
    let totalMemory = 0
    try {
      const os = require('node:os')
      freeMemory = os.freemem()
      totalMemory = os.totalmem()
    } catch (e) {
      // OS module not available
    }

    const stats = {
      memory: {
        heapUsed: heapStats.heapUsed,
        heapTotal: heapStats.heapTotal,
        external: heapStats.external,
        rss: heapStats.rss,
        free: freeMemory,
        total: totalMemory,
        containerLimit: config.detectedContainerLimit
      },
      limits: {
        maxQueryLimit: config.maxLimit,
        maxQueryLength: config.maxQueryLength,
        maxVectorDimensions: config.maxVectorDimensions,
        basis: config.limitBasis
      },
      config: {
        maxQueryLimit: this.config.maxQueryLimit,
        reservedQueryMemory: this.config.reservedQueryMemory
      },
      recommendations: [] as string[]
    }

    // Generate recommendations based on stats
    if (stats.limits.basis === 'freeMemory' && stats.memory.containerLimit) {
      stats.recommendations.push(
        `Container detected (${Math.round(stats.memory.containerLimit / 1024 / 1024)}MB) but limits based on free memory. ` +
        `Consider setting reservedQueryMemory config option for better limits.`
      )
    }

    if (stats.limits.maxQueryLimit < 5000 && stats.memory.containerLimit && stats.memory.containerLimit > 2 * 1024 * 1024 * 1024) {
      stats.recommendations.push(
        `Query limit is low (${stats.limits.maxQueryLimit}) despite ${Math.round(stats.memory.containerLimit / 1024 / 1024 / 1024)}GB container. ` +
        `Consider: new Brainy({ reservedQueryMemory: 1073741824 }) to reserve 1GB for queries.`
      )
    }

    if (stats.limits.basis === 'override') {
      stats.recommendations.push(
        `Using explicit maxQueryLimit override (${stats.limits.maxQueryLimit}). ` +
        `Auto-detection bypassed.`
      )
    }

    return stats
  }

  // ============= SUB-APIS =============

  /**
   * Natural Language Processing API
   */
  nlp(): NaturalLanguageProcessor {
    if (!this._nlp) {
      this._nlp = new NaturalLanguageProcessor(this)
    }
    return this._nlp
  }

  /**
   * Entity Extraction API - Neural extraction with NounType taxonomy
   *
   * Extracts entities from text using:
   * - Pattern-based candidate detection
   * - Embedding-based type classification
   * - Context-aware confidence scoring
   *
   * @param text - Text to extract entities from
   * @param options - Extraction options
   * @returns Array of extracted entities with types and confidence
   *
   * Fast heuristic ensemble (pattern + type-embedding + context), not a trained NER —
   * each candidate is typed by its own span (no cross-candidate bleed). Confidences are
   * approximate; pass `types` to constrain results when precision matters.
   *
   * @param text - Text to extract entities from
   * @param options - Extraction options
   * @returns Array of extracted entities with types and confidence
   *
   * @example
   * const entities = await brain.extract('Sarah Chen founded Acme Corp')
   * // [
   * //   { text: 'Sarah Chen', type: NounType.Person, confidence: 0.68 },
   * //   { text: 'Acme Corp', type: NounType.Organization, confidence: 0.85 }
   * // ]
   */
  async extract(
    text: string,
    options?: {
      types?: NounType[]
      confidence?: number
      includeVectors?: boolean
      neuralMatching?: boolean
    }
  ): Promise<ExtractedEntity[]> {
    if (!this._extractor) {
      this._extractor = new NeuralEntityExtractor(this)
    }
    return await this._extractor.extract(text, options)
  }

  /**
   * Extract entities from text (alias for extract())
   * Added for API clarity — `extractEntities()` reads more naturally at call sites
   *
   * Uses NeuralEntityExtractor with SmartExtractor ensemble (4-signal architecture):
   * - ExactMatch (40%) - Dictionary lookups
   * - Embedding (35%) - Semantic similarity
   * - Pattern (20%) - Regex patterns
   * - Context (5%) - Contextual hints
   *
   * @param text - Text to extract entities from
   * @param options - Extraction options
   * @returns Array of extracted entities with types and confidence scores
   *
   * @example
   * ```typescript
   * const entities = await brain.extractEntities('John Smith founded Acme Corp', {
   *   confidence: 0.7,
   *   types: [NounType.Person, NounType.Organization],
   *   neuralMatching: true
   * })
   * ```
   */
  async extractEntities(
    text: string,
    options?: {
      types?: NounType[]
      confidence?: number
      includeVectors?: boolean
      neuralMatching?: boolean
    }
  ): Promise<ExtractedEntity[]> {
    return this.extract(text, options)
  }

  /**
   * Extract concepts from text
   *
   * Simplified interface for concept/topic extraction
   * Returns only concept names as strings for easy metadata population
   *
   * @param text - Text to extract concepts from
   * @param options - Extraction options
   * @returns Array of concept names
   *
   * @example
   * const concepts = await brain.extractConcepts('Using OAuth for authentication')
   * // ['oauth', 'authentication']
   */
  async extractConcepts(
    text: string,
    options?: {
      confidence?: number
      limit?: number
    }
  ): Promise<string[]> {
    const entities = await this.extract(text, {
      types: [NounType.Concept],
      confidence: options?.confidence || 0.7,
      neuralMatching: true
    })

    // Deduplicate and normalize
    const conceptSet = new Set(entities.map(e => e.text.toLowerCase()))
    const concepts = Array.from(conceptSet)

    // Apply limit if specified
    return options?.limit ? concepts.slice(0, options.limit) : concepts
  }

  /**
   * Import files with intelligent extraction and dual storage (VFS + Knowledge Graph)
   *
   * Unified import system that:
   * - Auto-detects format (Excel, PDF, CSV, JSON, Markdown)
   * - Extracts entities with AI-powered name/type detection
   * - Infers semantic relationships from context
   * - Stores in both VFS (organized files) and Knowledge Graph (connected entities)
   * - Links VFS files to graph entities
   *
   * @since 4.0.0
   *
   * @example Quick Start (All AI features enabled by default)
   * ```typescript
   * const result = await brain.import('./glossary.xlsx')
   * // Auto-detects format, extracts entities, infers relationships
   * ```
   *
   * @example Full-Featured Import (v4.x)
   * ```typescript
   * const result = await brain.import('./data.xlsx', {
   *   // AI features
   *   enableNeuralExtraction: true,      // Extract entity names/metadata
   *   enableRelationshipInference: true, // Detect semantic relationships
   *   enableConceptExtraction: true,     // Extract types/concepts
   *
   *   // VFS features
   *   vfsPath: '/imports/my-data',       // Store in VFS directory
   *   groupBy: 'type',                   // Organize by entity type
   *   preserveSource: true,              // Keep original file
   *
   *   // Progress tracking (STANDARDIZED FOR ALL 7 FORMATS!)
   *   onProgress: (p) => {
   *     console.log(`[${p.stage}] ${p.message}`)
   *     console.log(`Entities: ${p.entities || 0}, Rels: ${p.relationships || 0}`)
   *     if (p.throughput) console.log(`Rate: ${p.throughput.toFixed(1)}/sec`)
   *   }
   * })
   * // THIS SAME HANDLER WORKS FOR CSV, PDF, Excel, JSON, Markdown, YAML, DOCX!
   * ```
   *
   * @example Universal Progress Handler
   * ```typescript
   * // ONE handler for ALL 7 formats - no format-specific code needed!
   * const universalProgress = (p) => {
   *   updateUI(p.stage, p.message, p.entities, p.relationships)
   * }
   *
   * await brain.import(csvBuffer, { onProgress: universalProgress })
   * await brain.import(pdfBuffer, { onProgress: universalProgress })
   * await brain.import(excelBuffer, { onProgress: universalProgress })
   * // Works for JSON, Markdown, YAML, DOCX too!
   * ```
   *
   * @example Performance Tuning (Large Files)
   * ```typescript
   * const result = await brain.import('./huge-file.csv', {
   *   enableDeduplication: false,  // Skip dedup for speed
   *   confidenceThreshold: 0.8,    // Higher threshold = fewer entities
   *   onProgress: (p) => console.log(`${p.processed}/${p.total}`)
   * })
   * ```
   *
   * @example Import from Buffer or Object
   * ```typescript
   * // From buffer
   * const result = await brain.import(buffer, { format: 'pdf' })
   *
   * // From object
   * const result = await brain.import({ entities: [...] })
   * ```
   *
   * @throws {Error} If invalid options are provided (v4.x breaking changes)
   *
   * @see {@link https://brainy.dev/docs/api/import API Documentation}
   * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
   * @see {@link https://brainy.dev/docs/guides/standard-import-progress Standard Progress API}
   *
   * @remarks
   * **⚠️ Breaking Changes from v3.x:**
   *
   * The import API was redesigned for clarity and better feature control.
   * Old v3.x option names are **no longer recognized** and will throw errors.
   *
   * **Option Changes:**
   * - ❌ `extractRelationships` → ✅ `enableRelationshipInference`
   * - ❌ `createFileStructure` → ✅ `vfsPath: '/your/path'`
   * - ❌ `autoDetect` → ✅ *(removed - always enabled)*
   * - ❌ `excelSheets` → ✅ *(removed - all sheets processed)*
   * - ❌ `pdfExtractTables` → ✅ *(removed - always enabled)*
   *
   * **New Options:**
   * - ✅ `enableNeuralExtraction` - Extract entity names via AI
   * - ✅ `enableConceptExtraction` - Extract entity types via AI
   * - ✅ `preserveSource` - Save original file in VFS
   *
   * **If you get an error:**
   * The error message includes migration instructions and examples.
   * See the complete migration guide for all details.
   *
   * **Why these changes?**
   * - Clearer option names (explicitly describe what they do)
   * - Separation of concerns (neural, relationships, VFS are separate)
   * - Better defaults (AI features enabled by default)
   * - Reduced confusion (removed redundant options)
   */
  async import(
    source: Buffer | string | object | PortableGraph,
    options?: {
      format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown' | 'yaml' | 'docx' | 'image'
      vfsPath?: string
      groupBy?: 'type' | 'sheet' | 'flat' | 'custom'
      customGrouping?: (entity: any) => string
      createEntities?: boolean
      createRelationships?: boolean
      preserveSource?: boolean
      enableNeuralExtraction?: boolean
      enableRelationshipInference?: boolean
      enableConceptExtraction?: boolean
      confidenceThreshold?: number
      onProgress?: (progress: {
        stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete'
        phase?: 'extraction' | 'relationships'
        message: string
        processed?: number
        current?: number
        total?: number
        entities?: number
        relationships?: number
        throughput?: number
        eta?: number
      }) => void
    } & Partial<ImportOptions>
  ): Promise<ImportResult | any> {
    // Portable graph round-trip: a PortableGraph document (produced by export()) is
    // restored via ONE atomic transaction — distinct from foreign-file ingestion below.
    if (isPortableGraph(source)) {
      return importGraph(this, this.storage, source, options as ImportOptions)
    }

    // Lazy load ImportCoordinator (foreign-file ingestion: CSV/PDF/Excel/JSON/…)
    const { ImportCoordinator } = await import('./import/ImportCoordinator.js')
    const coordinator = new ImportCoordinator(this)
    await coordinator.init()

    return await coordinator.import(source as Buffer | string | object, options)
  }

  /** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */
  private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator

  /**
   * The single brain-owned BackgroundDeduplicator, lazily constructed.
   *
   * Ownership matters here: the post-import dedup timer must outlive the
   * per-call ImportCoordinator but never the brain. One instance per brain
   * restores the intended cross-import debounce (per-coordinator instances
   * each armed their own timer, so the "debounce" never spanned imports) and
   * gives close() a handle to cancel pending work — a delete pass must never
   * fire against a closed brain.
   * @internal
   */
  async getBackgroundDeduplicator(): Promise<
    import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
  > {
    if (!this._backgroundDedup) {
      const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js')
      this._backgroundDedup = new BackgroundDeduplicator(this)
    }
    return this._backgroundDedup
  }

  /**
   * Virtual File System API - Knowledge Operating System
   *
   * Returns a cached VFS instance that is auto-initialized during brain.init().
   * No separate initialization needed!
   *
   * @example After import
   * ```typescript
   * await brain.import('./data.xlsx', { vfsPath: '/imports/data' })
   * // VFS ready immediately - no init() call needed!
   * const files = await brain.vfs.readdir('/imports/data')
   * ```
   *
   * @example Direct VFS usage
   * ```typescript
   * await brain.init()  // VFS auto-initialized here!
   * await brain.vfs.writeFile('/docs/readme.md', 'Hello World')
   * const content = await brain.vfs.readFile('/docs/readme.md')
   * ```
   *
   * **Pattern:** The VFS instance is cached, so multiple calls to brain.vfs
   * return the same instance. This ensures import and user code share state.
   *
   */
  get vfs(): VirtualFileSystem {
    if (!this._vfs) {
      // VFS is initialized during brain.init()
      // If not initialized yet, create instance but user should call brain.init() first
      this._vfs = new VirtualFileSystem(this)
    }
    // Warn if VFS accessed before init() completed
    if (!this._vfsInitialized && this.initialized) {
      console.warn('[Brainy] VFS accessed before initialization complete. Call await brain.init() first.')
    }
    return this._vfs
  }

  /**
   * Integration Hub for external tools (Excel, Power BI, Google Sheets)
   *
   * Provides HTTP endpoints that external tools can connect to:
   * - OData API for Excel Power Query, Power BI, Tableau
   * - REST API for Google Sheets custom functions
   * - SSE streaming for real-time dashboards
   * - Webhooks for push notifications
   *
   * Only available when `integrations: true` is set in config.
   *
   * @example Basic usage
   * ```typescript
   * const brain = new Brainy({ integrations: true })
   * await brain.init()
   *
   * // Get endpoint URLs
   * console.log(brain.hub.endpoints)
   * // { odata: '/odata', sheets: '/sheets', sse: '/events', webhooks: '/webhooks' }
   *
   * // Handle requests (use with Express, Hono, etc.)
   * app.all('/odata/*', async (req, res) => {
   *   const response = await brain.hub.handleRequest({
   *     method: req.method,
   *     path: req.path,
   *     query: req.query,
   *     headers: req.headers,
   *     body: req.body
   *   })
   *   res.status(response.status).set(response.headers).json(response.body)
   * })
   * ```
   *
   * @throws Error if integrations are not enabled in config
   */
  get hub(): IntegrationHub {
    if (!this._hub) {
      throw new Error(
        'Integration Hub not enabled. Set integrations: true in config:\n' +
        'new Brainy({ integrations: true })'
      )
    }
    return this._hub
  }

  /**
   * @description Subscribe to this brain's committed mutations — the
   * authoritative in-process change feed. Fires exactly once per affected
   * record for EVERY canonical write, regardless of origin: direct API calls,
   * batch methods, `transact()`, imports, the Virtual Filesystem, and
   * native-accelerated deployments all funnel through the same commit point
   * this feed is emitted from.
   *
   * Delivery contract (see {@link BrainyChangeEvent}):
   * - **Post-commit only** — an aborted write (e.g. a losing `ifRev`
   *   compare-and-swap) never emits.
   * - **Commit-ordered, asynchronous** — events arrive in the order writes
   *   became durable, dispatched in a microtask so a slow listener never
   *   delays a write. A throwing listener is isolated (logged, others
   *   unaffected).
   * - **Fully described** — `remove`/`unrelate` events carry the record's
   *   LAST committed state, and batch operations emit one event per item.
   * - **Fire-and-forget** — no replay or backpressure. Each event carries its
   *   committed `generation`, so catch-up after a gap can be built on
   *   {@link transactionLog} / {@link asOf}.
   * - **Zero overhead when unused** — with no subscribers the write path does
   *   no event work at all.
   *
   * @param listener - Called once per committed mutation.
   * @returns An unsubscribe function — call it when done (e.g. on teardown of
   *   a pooled instance) to stop delivery.
   * @example
   * const off = brain.onChange((e) => {
   *   if (e.kind === 'entity') console.log(e.op, e.entity?.type, e.id)
   * })
   * await brain.add({ data: 'hello', type: 'document' }) // → "add document <id>"
   * off()
   */
  onChange(listener: ChangeListener): () => void {
    return this._changeFeed.subscribe(listener)
  }

  /**
   * Get Triple Intelligence System
   * Advanced pattern recognition and relationship analysis
   */
  getTripleIntelligence(): TripleIntelligenceSystem {
    if (!this._tripleIntelligence) {
      // Use core components directly - no lazy loading needed.
      // TripleIntelligenceSystem speaks UUIDs; adapt the BigInt graph-index
      // contract through the coordinator's UUID-level helper so the int
      // conversion stays at this boundary.
      this._tripleIntelligence = new TripleIntelligenceSystem(
        this.metadataIndex,
        this.index,
        {
          getNeighbors: (id: string, direction?: 'in' | 'out' | 'both') =>
            this.getNeighborUuids(id, { direction }),
          size: () => this.graphIndex.size()
        },
        async (text: string) => this.embedder(text),
        this.storage
      )
    }
    return this._tripleIntelligence
  }

  // ============= METADATA INTELLIGENCE API =============
  
  /**
   * Get all indexed field names currently in the metadata index
   * Essential for dynamic query building and NLP field discovery
   */
  async getAvailableFields(): Promise<string[]> {
    await this.ensureInitialized()
    return this.metadataIndex.getFilterFields()
  }
  
  /**
   * Get field statistics including cardinality and query patterns
   * Used for query optimization and understanding data distribution
   */
  async getFieldStatistics(): Promise<Map<string, any>> {
    await this.ensureInitialized()
    return this.metadataIndex.getFieldStatistics()
  }
  
  /**
   * Get fields sorted by cardinality for optimal filtering
   * Lower cardinality fields are better for initial filtering
   */
  async getFieldsWithCardinality(): Promise<Array<{
    field: string
    cardinality: number
    distribution: string
  }>> {
    await this.ensureInitialized()
    return this.metadataIndex.getFieldsWithCardinality()
  }
  
  /**
   * Get optimal query plan for a given set of filters
   * Returns field processing order and estimated cost
   */
  async getOptimalQueryPlan(filters: Record<string, any>): Promise<{
    strategy: 'exact' | 'range' | 'hybrid'
    fieldOrder: string[]
    estimatedCost: number
  }> {
    await this.ensureInitialized()
    return this.metadataIndex.getOptimalQueryPlan(filters)
  }
  
  /**
   * Get filter values for a specific field (for UI dropdowns, etc)
   */
  async getFieldValues(field: string): Promise<string[]> {
    await this.ensureInitialized()
    return this.metadataIndex.getFilterValues(field)
  }
  
  /**
   * Get fields that commonly appear with a specific entity type
   * Essential for type-aware NLP parsing
   */
  async getFieldsForType(nounType: NounType): Promise<Array<{
    field: string
    affinity: number
    occurrences: number
    totalEntities: number
  }>> {
    await this.ensureInitialized()
    return this.metadataIndex.getFieldsForType(nounType)
  }
  

  /**
   * Create a streaming pipeline
   */
  stream() {
    const { Pipeline } = require('./streaming/pipeline.js')
    return new Pipeline(this)
  }

  /**
   * Get insights about the data
   */
  async insights(): Promise<{
    entities: number
    relationships: number
    types: Record<string, number>
    services: string[]
    density: number
  }> {
    await this.ensureInitialized()
    
    // O(1) entity counting using existing MetadataIndexManager
    const entities = this.metadataIndex.getTotalEntityCount()

    // O(1) count by type using existing index tracking
    const typeCountsMap = this.metadataIndex.getAllEntityCounts()
    const types: Record<string, number> = Object.fromEntries(typeCountsMap)

    // O(1) relationships count using GraphAdjacencyIndex
    const relationships = this.graphIndex.getTotalRelationshipCount()
    
    // Get unique services - O(log n) using index
    const serviceValues = await this.metadataIndex.getFilterValues('service')
    const services = serviceValues.filter(Boolean)
    
    // Calculate density (relationships per entity)
    const density = entities > 0 ? relationships / entities : 0
    
    return {
      entities,
      relationships,
      types,
      services,
      density
    }
  }

  /**
   * @description Stamp every projection's watermark with the store's
   * current committed generation — the door BOTH {@link flush} and {@link
   * close} open right before persisting, so EITHER path leaves a stamped,
   * `'adopt'`-verdicting artifact on disk (stamp-after-data still holds
   * inside each owner: this only hands the generation over — the owner's
   * OWN flush is what durably writes the stamp, LAST). Before this method
   * existed, `close()` had its own separate flush fan-out that never
   * stamped, so a `close()` without a preceding explicit `flush()` left
   * every projection unstamped — a real, closed store that legitimately
   * verdicts `'rescan'` on its very next open (not a bug in the verdict,
   * a gap in `close()`'s persistence completeness that this closes).
   * No `committedGeneration` capability, or a replacement provider that
   * doesn't carry the stamp method (a native pair swaps these managers) =
   * no stamp = the owner's verdict machinery treats the artifact as
   * legacy — never a flush/close crash either way.
   */
  private stampProjectionWatermarks(): void {
    const wmGen = this.storage?.committedGeneration?.() ?? null
    if (wmGen === null) return
    ;(this.metadataIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen)
    ;(this.index as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen)
    ;(this.graphIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen)
  }

  /**
   * Flush all indexes and caches to persistent storage
   * CRITICAL FIX: Ensures data survives server restarts
   *
   * Flushes all 4 core indexes:
   * 1. Storage counts (entity/verb counts by type)
   * 2. Metadata index (field indexes + EntityIdMapper)
   * 3. Graph adjacency index (relationship cache)
   * 4. HNSW vector index (deferred dirty nodes)
   *
   * @example
   * // Flush after bulk operations
   * await brain.import('./data.xlsx')
   * await brain.flush()
   *
   * // Flush before shutdown
   * process.on('SIGTERM', async () => {
   *   await brain.flush()
   *   process.exit(0)
   * })
   */
  async flush(): Promise<void> {
    await this.ensureInitialized()

    // Read-only instances have no buffered writes to flush. close() may call
    // flush() defensively, so we early-return instead of throwing.
    if (this.isReadOnly) {
      return
    }

    // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been
    // committed since the last flush, so every step below would re-persist
    // state identical to what is already on disk — provider flushes, the
    // watermark stamps, the generation counter, the entity-tree stamp — and
    // print two lines announcing it. The witness is set by every committed
    // write (see noteWriteForPersistence) and cleared here; a write landing
    // DURING this flush sets it again, so it is never lost — the next flush
    // does that write's work. This makes an unexplained flush FREE; it does
    // not explain one (see _dirtySinceLastFlush).
    if (!this._dirtySinceLastFlush) {
      return
    }
    this._dirtySinceLastFlush = false
    // An explicit flush IS a flush: tell the cadence so, or the very next
    // write sees "30s since the last flush" (the cadence only counted its
    // own) and kicks a background flush that has nothing left to do, and the
    // idle timer fires two seconds later over writes this flush already
    // persisted.
    this._persistLastFlushAt = Date.now()
    this._persistDirtyWrites = 0

    console.log('Flushing Brainy indexes and caches to disk...')
    const startTime = Date.now()

    // 8.0 MVCC: persist the in-memory pending single-op generation history to
    // disk first (async group-commit), so a flush makes every single-op write's
    // history durable, not just the live data.
    await this.generationStore.flushPendingSingleOps()

    // Flush all components in parallel for performance
    // Watermark stamps ride every flush fan-out — see stampProjectionWatermarks().
    this.stampProjectionWatermarks()
    await Promise.all([
      // 1. Flush storage adapter counts (entity/verb counts by type)
      (async () => {
        if (this.storage && typeof this.storage.flushCounts === 'function') {
          await this.storage.flushCounts()
        }
      })(),

      // 2. Flush metadata index (field indexes + EntityIdMapper)
      this.metadataIndex.flush(),

      // 3. Flush graph adjacency index (relationship cache + LSM trees)
      this.graphIndex.flush(),

      // 4. Flush HNSW dirty nodes (deferred persistence mode)
      (async () => {
        if (this.index && typeof this.index.flush === 'function') {
          await this.index.flush()
        }
      })(),

      // 5. Persist the generation counter (8.0 MVCC — coalesced single-op
      //    bumps become durable on every explicit flush)
      this.generationStore.persistCounterNow(),

      // 6. Persist aggregation state, stamped at the committed generation
      //    (BRAINY-PROD-LATENCY-TRIAD / SELF-ENGINE-LIFECYCLE-SPRINT ask (a)):
      //    aggregation used to persist ONLY at close(), so a long-lived
      //    writer that flushes but never closes — the primary production
      //    shape — left its stamp behind after every write window, and any
      //    unclean exit forced a WHOLE-STORE backfill walk on the next
      //    first stats call (measured >60s and door-starving on a 9k-row
      //    production brain). Flushing here keeps the stamp current, so a
      //    reopen adopts (or incrementally catches up) instead of rescanning.
      (async () => {
        if (this._aggregationIndex) {
          await this._aggregationIndex.flush()
        }
      })()
    ])

    // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY
    // work — it must cost what this window's deltas cost, never what the
    // history backlog costs. Auto-compaction (a MAINTENANCE concern) runs at
    // close() and via explicit compactHistory(); a production deployment
    // measured reclaim-on-flush stalling single writes for 25-191s under
    // memory pressure, which is exactly the class this separation ends.

    // 6. Stamp the entity tree: which source generation the canonical tree
    //    reflects + the rollup invariants that verify it whole (the counters
    //    persisted in step 1). Written at flush boundaries — the tree tracks
    //    every commit by construction, so the stamp is a durable checkpoint,
    //    not a per-commit cost. Open compares stamp vs log head + rollups.
    await this.stampEntityTree()

    const elapsed = Date.now() - startTime

    console.log(`All indexes flushed to disk in ${elapsed}ms`)
  }

  /**
   * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the
   * committed generation the canonical tree reflects — equal by construction,
   * since the tree is written by the commit itself) plus the rollup invariants
   * (entity/relationship counts) that verify the tree whole where per-file
   * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp};
   * healed by `repairIndex()`, whose unconditional recount rebuilds the
   * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write
   * fault warns loudly but never fails the flush that carried real data.
   *
   * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is
   * the ALLOCATED counter — a number a write in flight has claimed and may
   * never commit. Stamping it made the stamp's generation label a claim about
   * counts it was not taken at, and every crash inside a write window then
   * produced a spurious verdict at the next open: either `sourceGeneration N
   * is ahead of the log head N-1` (the allocated generation died with the
   * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the
   * recovery fold folded facts the stamp's counts predate). MEASURED on the
   * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on
   * a coherent store raised one of those two verdicts, each of them naming
   * `repairIndex()` — a whole-store recount — as the cure for nothing.
   */
  private async stampEntityTree(): Promise<void> {
    if (this.isReadOnly) return
    try {
      const [nounCount, verbCount] = await Promise.all([
        this.storage.getNounCount(),
        this.storage.getVerbCount()
      ])
      await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
        family: 'entity-tree',
        sourceGeneration: this.generationStore.committedGeneration(),
        members: { mode: 'rollup', invariants: { nounCount, verbCount } }
      })
    } catch (error) {
      prodLog.warn(
        `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` +
          `next successful flush): ${(error as Error).message}`
      )
    }
  }

  /**
   * @description Open-time coherence check for the entity tree's family stamp:
   * compare `sourceGeneration` against the store's COMMITTED generation and
   * the stamped rollup invariants against the live counters. Verdicts:
   * - `coherent` / `absent` (legacy store; first flush stamps) → silent.
   * - `behind` → benign for the tree (it is written BY the commit; only the
   *   stamp is stale — a crash landed between commit and flush). Refreshed at
   *   the next flush.
   * - `torn` → a TORN GENERATION-LOG TAIL, handled by
   *   {@link demoteTornEntityTreeStamp}: terminal, never a wait.
   * - `incoherent` → LOUD: the tree or its counters diverged from what was
   *   stamped — `repairIndex()` recounts from canonical and re-stamps.
   * Never blocks open; a fault reading the stamp is surfaced as unverifiable,
   * never conflated with absence.
   *
   * THE COMPARISON IS AGAINST `committedGeneration()`, matching what
   * {@link stampEntityTree} writes and what every other open-time watermark in
   * this class already reasons about (the fact-scan capability, the metadata /
   * graph / HNSW watermark verdicts). Comparing against the allocated counter
   * was the one place that disagreed, and disagreeing was the whole defect.
   */
  private async verifyEntityTreeStamp(): Promise<void> {
    let stamp: FamilyStamp | null
    try {
      stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH)
    } catch (error) {
      prodLog.warn(
        `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` +
          `${(error as Error).message}`
      )
      return
    }
    const [nounCount, verbCount] = await Promise.all([
      this.storage.getNounCount(),
      this.storage.getVerbCount()
    ])
    const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), {
      nounCount,
      verbCount
    })
    if (verdict.state === 'torn') {
      await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, {
        nounCount,
        verbCount
      })
    } else if (verdict.state === 'incoherent') {
      prodLog.warn(
        `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` +
          `The canonical tree or its counters diverged from the stamped state — run ` +
          `brain.repairIndex() to recount from canonical and re-stamp.`
      )
    } else if (verdict.state === 'behind') {
      prodLog.debug(
        `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` +
          `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.`
      )
    }
  }

  /**
   * @description THE TERMINAL VERDICT for a torn generation-log tail.
   *
   * A stamp whose `sourceGeneration` sits ABOVE the store's committed
   * watermark witnesses a generation that is not in the log: the stamp's fsync
   * outlived the tail's. By the time this runs, log-authority recovery has
   * already folded every intact fact above the manifest and advanced the
   * watermark to cover them — so if the stamp is STILL ahead, the generation
   * it names is not merely late, it is GONE. There is nothing to wait for.
   *
   * That is the whole point of this method. A field report of this class
   * (single-process store, abrupt termination mid-fold) described a reopen
   * that narrated the tear and then held 100% CPU with zero log growth for
   * eight minutes before an operator wiped the directory. A recovery that
   * cannot say what it is waiting for has no business spinning; the honest
   * answer here is a verdict, taken now, at O(1) cost.
   *
   * WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is
   * discarded rather than believed: the stamped counts describe a generation
   * that never became durable, and comparing them against live counters can
   * only produce noise. The tree itself is not in question (it IS canonical —
   * every commit writes it, and the fold re-applied every after-image the log
   * still holds), so the demotion is a re-derivation of this family's verified
   * surface at the generation the store can actually show:
   *
   * - WRITER open → re-stamp at `committedGeneration()` from the live
   *   counters — exactly what the next flush would write, taken now so the
   *   tear cannot re-narrate on every subsequent open. Both count sets are
   *   logged so an operator can see whether anything really moved.
   * - READER open → a reader cannot re-stamp. Narrate the same terminal
   *   verdict with the named cure and carry on serving; a read-only inspector
   *   is never locked out of a store, and never left waiting either.
   *
   * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any
   * external progress signal — the two counter reads and one stamp write are
   * the entire cost, and none of them scales with the store.
   */
  private async demoteTornEntityTreeStamp(
    stamp: FamilyStamp,
    stampSource: number,
    head: number,
    observed: { nounCount: number; verbCount: number }
  ): Promise<void> {
    const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {}
    const detail =
      `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` +
      `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` +
      `generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` +
      `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` +
      `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.`

    if (this.isReadOnly) {
      prodLog.warn(
        `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` +
          `family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` +
          `is the truth this stamp only describes). Cure: open the store with a writer, or run ` +
          `brain.repairIndex() there, to recount from canonical and re-stamp.`
      )
      return
    }

    const startedAt = Date.now()
    try {
      await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
        family: 'entity-tree',
        sourceGeneration: head,
        members: {
          mode: 'rollup',
          invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount }
        }
      })
      prodLog.warn(
        `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` +
          `from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` +
          `observed counts above look wrong for your data, run brain.repairIndex() to recount ` +
          `from canonical.`
      )
    } catch (error) {
      prodLog.warn(
        `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` +
          `narrate again at the next open, which is the honest outcome; the store still serves ` +
          `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.`
      )
    }
  }

  /**
   * Ask the writer process serving this data directory to flush its in-memory
   * indexes to disk, so a read-only inspector can observe fresh state.
   *
   * - **Same-process call** (this instance owns the writer lock): equivalent
   *   to `this.flush()`.
   * - **Different process** (typical inspector flow): writes a request file
   *   into the lock directory and polls for an ack file. The writer's
   *   flush-request watcher (started in `init()` for writer instances) sees
   *   the request, calls `flush()`, and writes the ack.
   * - **No writer running**: times out and returns `false`. Callers can
   *   proceed with last-flushed disk state and warn the operator.
   *
   * @param options.timeoutMs - How long to wait for an ack (default 5000ms).
   * @returns `true` if the writer flushed in response to this request,
   *   `false` on timeout (no writer or unresponsive).
   *
   * @example
   * ```typescript
   * const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } })
   * const fresh = await reader.requestFlush({ timeoutMs: 3000 })
   * if (!fresh) {
   *   console.warn('Writer did not respond; results reflect last natural flush.')
   * }
   * const bookings = await reader.find({ where: { entityType: 'booking' } })
   * ```
   */
  async requestFlush(options?: { timeoutMs?: number }): Promise<boolean> {
    await this.ensureInitialized()
    const timeoutMs = options?.timeoutMs ?? 5000

    // In-process shortcut: if this instance can write, just flush directly.
    if (!this.isReadOnly) {
      await this.flush()
      return true
    }

    if (typeof this.storage.requestFlushOverFilesystem !== 'function') {
      return false
    }
    return this.storage.requestFlushOverFilesystem(timeoutMs)
  }

  /**
   * @description The per-projection catch-up gauges served on
   * `getIndexStatus().projections` (both the initialized and the
   * pre-init snapshot — the numbers are safe to read at any lifecycle
   * stage). Semantic reports the live deferred-embed backlog; metadata and
   * graph are synchronous today (updated inside the write path);
   * aggregation reports its rescan/catch-up backlogs (zero when the
   * aggregation engine was never engaged).
   */
  private projectionGauges(): {
    semantic: { pendingEmbeds: number }
    metadata: { synchronous: true }
    graph: { synchronous: true }
    aggregation: { pendingBackfills: number; pendingCatchUps: number }
  } {
    return {
      semantic: { pendingEmbeds: this._pendingEmbedIds.size },
      metadata: { synchronous: true },
      graph: { synchronous: true },
      aggregation: {
        pendingBackfills: this._aggregationIndex?.getPendingBackfills().length ?? 0,
        pendingCatchUps: this._aggregationIndex?.getPendingCatchUps().length ?? 0
      }
    }
  }

  /**
   * Get index loading status (diagnostic)
   *
   * Returns detailed information about index population state. Useful for
   * debugging empty query results or performance troubleshooting.
   *
   * @example
   * ```typescript
   * const status = await brain.getIndexStatus()
   * console.log(`HNSW Index: ${status.hnswIndex.size} entities`)
   * console.log(`Metadata Index: ${status.metadataIndex.entries} entries`)
   * console.log(`Graph Index: ${status.graphIndex.relationships} relationships`)
   * console.log(`Pending embeds: ${status.projections.semantic.pendingEmbeds}`)
   * console.log(`Index build completed at open: ${status.lazyRebuildCompleted}`)
   * ```
   */

  /**
   * The provider-seam belt for filter reads: whatever manager serves
   * getIdsForFilter (the JS twin or a native replacement), a field refusal
   * crossing this seam is normalized to BRAINY'S UnresolvableFieldError —
   * one class identity for consumers, never a foreign twin that fails
   * instanceof. All other errors pass through untouched.
   */
  private async filterIdsBelted(
    filter: unknown,
    opts?: { limit?: number; offset?: number }
  ): Promise<string[]> {
    // READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
    // index read funnels through this helper, so the gate here makes
    // serve-while-not-ready UNREPRESENTABLE — a production store once acked
    // writes while every non-find() read served empty from a not-ready
    // provider for 15 minutes. A CHECK only — it never builds; throws a typed
    // NotReady error if a provider's health report says it isn't serving.
    this.ensureIndexesLoaded(['metadata'])
    try {
      return await this.metadataIndex.getIdsForFilter(filter, opts)
    } catch (err) {
      const normalized = asBrainyFieldRefusal(err)
      if (normalized) throw normalized
      throw err
    }
  }

  async getIndexStatus(): Promise<{
    initialized: boolean
    /** `true` once open()'s index-build-if-needed step has run. Named for API
     *  compatibility with the retired first-query lazy-build path; a needed
     *  rebuild now always runs at open, never deferred to a read, so this is
     *  simply `initialized`'s index-build counterpart. */
    lazyRebuildCompleted: boolean
    /** Deferred embeds not yet landed (MT5) — the eventual-vector-index backlog. */
    pendingEmbeds: number
    /** Per-projection catch-up gauges — the honest numbers behind
     *  {@link waitForIndexed}. `synchronous: true` marks projections updated
     *  inside the write path today: their barrier leg resolves immediately by
     *  design, and the flag becomes a real backlog gauge when the
     *  log-authority read path makes them asynchronous. */
    projections: {
      /** The deferred-embedding backlog (same number as the top-level
       *  `pendingEmbeds`, which stays for compat). */
      semantic: { pendingEmbeds: number }
      metadata: { synchronous: true }
      graph: { synchronous: true }
      aggregation: {
        /** Aggregates flagged for a full rescan of existing entities
         *  (drained on the next aggregate query). */
        pendingBackfills: number
        /** Aggregates adopted behind the watermark, with exact missing
         *  windows still to reconcile. */
        pendingCatchUps: number
      }
    }
    disableAutoRebuild: boolean
    /** `true` while a native provider runs the one-time 7.x → 8.0 rebuild LOCK.
     *  A readiness probe should map this to HTTP 503 + Retry-After (transiently
     *  not-ready), NOT 200-ready and NOT 500-broken. Never gated by the lock. */
    migrating: boolean
    /** Structured progress while {@link migrating}; absent otherwise. */
    migration?: MigrationProgress
    /** `true` when a non-fatal index rebuild failed at init() (degraded — queries
     *  may be incomplete). Folds in the same `_indexRebuildFailed` signal that
     *  {@link validateIndexConsistency} / {@link checkHealth} already expose. */
    rebuildFailed: boolean
    /** The rebuild failure message when {@link rebuildFailed}; absent otherwise. */
    rebuildError?: string
    /** Count of records committed via adopt-forward failed-rollback recovery whose
     *  derived index may be incomplete until repairIndex() (the 8.2.6 degraded
     *  set). Non-zero = degraded — a readiness probe should not report 200-ready. */
    degradedIds: number
    hnswIndex: {
      size: number
      /** Honest "serving" — the provider's `isReady()` when exposed, else `size>0`. */
      populated: boolean
      /** The provider's honest `isReady()` signal; absent when unexposed. */
      ready?: boolean
    }
    metadataIndex: {
      entries: number
      populated: boolean
      ready?: boolean
    }
    graphIndex: {
      relationships: number
      populated: boolean
      ready?: boolean
    }
    storage: {
      totalEntities: number
    }
  }> {
    // Callable before init() — a readiness/liveness probe may fire during
    // construction or a fired-not-awaited init(). Report a safe not-initialized
    // snapshot rather than dereferencing providers that init() has not assigned.
    if (!this.initialized || this.metadataIndex == null || this.index == null || this.graphIndex == null) {
      return {
        initialized: false,
        lazyRebuildCompleted: this.lazyRebuildCompleted,
        pendingEmbeds: this._pendingEmbedIds.size,
        projections: this.projectionGauges(),
        disableAutoRebuild: this.config.disableAutoRebuild || false,
        migrating: false,
        rebuildFailed: this._indexRebuildFailed != null,
        ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}),
        degradedIds: this._indexDegradedIds.size,
        hnswIndex: { size: 0, populated: false },
        metadataIndex: { entries: 0, populated: false },
        graphIndex: { relationships: 0, populated: false },
        storage: { totalEntities: 0 }
      }
    }

    const metadataStats = await this.metadataIndex.getStats()
    const hnswSize = this.index.size()
    const graphSize = await this.graphIndex.size()

    // Honest readiness: when a provider exposes isReady(), it is the truth of
    // whether the index actually SERVES (a native index can report a non-zero
    // size/count yet not have loaded its serving structure). `populated` reflects
    // that honest signal when present, falling back to size>0 only for providers
    // that do not expose isReady(). Mirrors validateIndexConsistency/checkHealth,
    // which already fold in the same signals.
    const readyOf = (p: unknown): boolean | undefined => {
      const r = assessIndexReadiness(p)
      return r === 'unknown' ? undefined : r === 'ready'
    }
    const hnswReady = readyOf(this.index)
    const metadataReady = readyOf(this.metadataIndex)
    const graphReady = readyOf(this.graphIndex)

    // Check storage entity count
    let storageEntityCount = 0
    try {
      const entities = await this.storage.getNouns({ pagination: { limit: 1 } })
      storageEntityCount = entities.totalCount || 0
    } catch (e) {
      // Ignore errors
    }

    return {
      initialized: this.initialized,
      lazyRebuildCompleted: this.lazyRebuildCompleted,
        pendingEmbeds: this._pendingEmbedIds.size,
      projections: this.projectionGauges(),
      disableAutoRebuild: this.config.disableAutoRebuild || false,
      // A non-fatal index-rebuild failure recorded at init(), or adopt-forward
      // degraded ids, are degraded states (queries may be incomplete) — surface
      // them here alongside the same signals validateIndexConsistency()/
      // checkHealth() already expose, so a readiness probe never reports 200-ready
      // over a known-degraded index.
      rebuildFailed: this._indexRebuildFailed != null,
      ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}),
      degradedIds: this._indexDegradedIds.size,
      ...this.migrationSnapshot(),
      hnswIndex: {
        size: hnswSize,
        populated: hnswReady ?? hnswSize > 0,
        ...(hnswReady !== undefined ? { ready: hnswReady } : {})
      },
      metadataIndex: {
        entries: metadataStats.totalEntries,
        populated: metadataReady ?? metadataStats.totalEntries > 0,
        ...(metadataReady !== undefined ? { ready: metadataReady } : {})
      },
      graphIndex: {
        relationships: graphSize,
        populated: graphReady ?? graphSize > 0,
        ...(graphReady !== undefined ? { ready: graphReady } : {})
      },
      storage: {
        totalEntities: storageEntityCount
      }
    }
  }

  /**
   * Run a battery of cheap invariant checks suitable for an operator-facing
   * health probe. Reports counts of suspicious states alongside the basic
   * size invariants — designed so an incident responder can spot the typical
   * failure modes (mismatched index sizes, lots of `_seeded` records hanging
   * around, missing field stats) without scanning the whole store.
   *
   * @example
   * ```typescript
   * const h = await brain.health()
   * for (const c of h.checks) {
   *   console.log(`${c.status === 'pass' ? '✓' : '!'} ${c.name}: ${c.message}`)
   * }
   * ```
   */
  async health(): Promise<{
    overall: 'pass' | 'warn' | 'fail'
    checks: Array<{
      name: string
      status: 'pass' | 'warn' | 'fail'
      message: string
      details?: Record<string, unknown>
    }>
  }> {
    // Lock-exempt: an operator must be able to probe health WHILE the brain
    // upgrades — that is the whole point of an observable migration.
    await this.ensureInitialized({ bypassMigrationLock: true })
    const checks: Array<{
      name: string
      status: 'pass' | 'warn' | 'fail'
      message: string
      details?: Record<string, unknown>
    }> = []

    // Migration LOCK (#18): while a native provider rebuilds the derived indexes,
    // the parity/field checks below would report transient mismatches that read
    // as failures but are not — surface the upgrade as the single honest signal.
    // `warn` (not `fail`) so a readiness probe treats it as "not ready yet, retry".
    const migSnap = this.migrationSnapshot()
    if (migSnap.migrating) {
      const pct = migSnap.migration?.percent
      return {
        overall: 'warn',
        checks: [
          {
            name: 'migration',
            status: 'warn',
            message:
              `Brain is upgrading (7.x → 8.0)${pct !== undefined ? `, ${Math.round(pct)}% complete` : ''}. ` +
              'Reads and writes are blocked until it completes; retry shortly.',
            details: { ...migSnap.migration }
          }
        ]
      }
    }

    const hnswSize = this.index.size()
    const metadataStats = await this.metadataIndex.getStats()
    const graphSize = await this.graphIndex.size()

    // 1. Index size parity. HNSW must hold one node per VECTORED noun — the
    // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw
    // metadata-entry count: every store's VFS root is PERMANENTLY unvectored
    // (`vector: []` by design — a zero-norm/empty vector never crosses into
    // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s
    // matching belts), and a not-yet-landed deferred embed is unvectored
    // too. Comparing against total entries counted the always-unvectored
    // root as a permanent 1-node "drift" on every VFS-having store — a false
    // warn on an otherwise perfectly healthy handoff. `vectors.all` is
    // already the documented coverage denominator for exactly this
    // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the
    // metadata-entry count when the ledger is unavailable or suspect (a
    // storage adapter without the optional hook, or an unrecounted store) —
    // never worse than the prior behavior in that case.
    const vectorLedgerForParity = await this.storage.getCanonicalCounts?.()
    const vectorParityTarget =
      vectorLedgerForParity && !vectorLedgerForParity.suspect
        ? vectorLedgerForParity.vectors.all
        : metadataStats.totalEntries
    if (hnswSize === vectorParityTarget) {
      checks.push({
        name: 'index-parity',
        status: 'pass',
        message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`,
        details: {
          hnswSize,
          vectoredNouns: vectorParityTarget,
          metadataEntries: metadataStats.totalEntries,
          graphRelationships: graphSize
        }
      })
    } else {
      const drift = Math.abs(hnswSize - vectorParityTarget)
      checks.push({
        name: 'index-parity',
        status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn',
        message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`,
        details: {
          hnswSize,
          vectoredNouns: vectorParityTarget,
          metadataEntries: metadataStats.totalEntries,
          drift
        }
      })
    }

    // 2. Field registry sanity. Empty field set + non-zero entities = stale reader.
    let fieldCount = 0
    try {
      if (typeof this.metadataIndex.getFieldStatistics === 'function') {
        const fieldStats = await this.metadataIndex.getFieldStatistics()
        fieldCount = fieldStats.size
      }
    } catch {
      // ignore — metadata index doesn't expose stats
    }
    if (metadataStats.totalEntries > 0 && fieldCount === 0) {
      checks.push({
        name: 'field-registry',
        status: 'warn',
        message: `${metadataStats.totalEntries} entities present but the field registry is empty. Reader may be stale; consider requestFlush().`,
        details: { entities: metadataStats.totalEntries, fields: fieldCount }
      })
    } else {
      checks.push({
        name: 'field-registry',
        status: 'pass',
        message: `${fieldCount} fields registered for ${metadataStats.totalEntries} entities.`,
        details: { fields: fieldCount, entities: metadataStats.totalEntries }
      })
    }

    // 3. Seeded entity sweep — operators often want to know if demo seed data
    // is still in a brain that's also serving real traffic (a common root
    // cause of duplicate-ID and stale-content bugs). Uses the metadata index,
    // not a full scan.
    let seededIds: string[] = []
    try {
      seededIds = await this.metadataIndex.getIds('_seeded', true)
    } catch {
      // ignore — field may not be indexed
    }
    if (seededIds.length > 0) {
      checks.push({
        name: 'seeded-records',
        status: 'warn',
        message: `${seededIds.length} entities tagged _seeded:true. Verify this is the demo data you expect.`,
        details: { count: seededIds.length, sample: seededIds.slice(0, 5) }
      })
    } else {
      checks.push({
        name: 'seeded-records',
        status: 'pass',
        message: 'No _seeded:true entities found.'
      })
    }

    // 4. Lock heartbeat freshness — only meaningful for readers inspecting a
    // running writer. If the lock exists but the heartbeat is old, the writer
    // probably crashed.
    if (typeof this.storage.readWriterLock === 'function') {
      const lock = await this.storage.readWriterLock()
      if (lock) {
        const age = Date.now() - new Date(lock.lastHeartbeat).getTime()
        if (age > 60_000) {
          checks.push({
            name: 'writer-heartbeat',
            status: 'warn',
            message: `Writer lock heartbeat is ${Math.round(age / 1000)}s old (PID ${lock.pid} on ${lock.hostname}). Writer may be hung.`,
            details: { lock, ageMs: age }
          })
        } else {
          checks.push({
            name: 'writer-heartbeat',
            status: 'pass',
            message: `Writer healthy (PID ${lock.pid} on ${lock.hostname}, heartbeat ${Math.round(age / 1000)}s ago).`,
            details: { lock }
          })
        }
      }
    }

    const worst = checks.reduce<'pass' | 'warn' | 'fail'>((acc, c) => {
      if (c.status === 'fail') return 'fail'
      if (c.status === 'warn' && acc !== 'fail') return 'warn'
      return acc
    }, 'pass')

    return { overall: worst, checks }
  }

  /**
   * Explain how a `find` query's `where` clause will be served. For each
   * field, returns whether it will hit the column store (best), a sparse
   * chunked index (legacy fallback), or has no index entries at all (silently
   * empty result — usually a bug or a stale reader). Designed to be the very
   * first thing an operator runs when `find()` returns surprising results.
   *
   * @example
   * ```typescript
   * const plan = await brain.explain({ where: { entityType: 'booking', status: 'paid' } })
   * for (const f of plan.fieldPlan) {
   *   console.log(`${f.field} -> ${f.path}: ${f.notes ?? ''}`)
   * }
   * ```
   */
  async explain(params: FindParams<T>): Promise<{
    query: FindParams<T>
    fieldPlan: Array<{ field: string; path: 'column-store' | 'sparse-chunked' | 'none'; notes?: string }>
    warnings: string[]
  }> {
    await this.ensureInitialized()
    const fieldPlan: Array<{ field: string; path: 'column-store' | 'sparse-chunked' | 'none'; notes?: string }> = []
    const warnings: string[] = []

    const where = params?.where
    if (where && typeof where === 'object') {
      for (const field of Object.keys(where)) {
        const result = await this.metadataIndex.explainField(field)
        fieldPlan.push({ field, path: result.path, notes: result.notes })
        if (result.path === 'none') {
          warnings.push(
            `Field "${field}" has no index entries. find() will return [] silently. ` +
            `Run brain.requestFlush() or check the writer's field registry.`
          )
        }
      }
    } else {
      warnings.push('No `where` clause provided; nothing to explain.')
    }

    return { query: params, fieldPlan, warnings }
  }

  /**
   * Operator-facing summary of what's in this Brainy store. Designed to be
   * the first thing a human runs during an incident: counts, mode, lock owner,
   * indexed field list, index health flags.
   *
   * Safe to call on a read-only instance. All counts come from already-loaded
   * indexes (no extra storage scans), so this is fast (<10ms typical).
   *
   * @example
   * ```typescript
   * const s = await brain.stats()
   * console.log(`${s.entityCount} entities (${Object.entries(s.entitiesByType).map(([t,n])=>`${t}:${n}`).join(', ')})`)
   * if (s.writerLock) {
   *   console.log(`Writer PID ${s.writerLock.pid} on ${s.writerLock.hostname}`)
   * }
   * ```
   */
  async stats(): Promise<BrainyStats> {
    await this.ensureInitialized()

    const { NounTypeEnum, VerbTypeEnum } = await import('./types/graphTypes.js')
    const nounCounts = typeof this.storage.getNounCountsByType === 'function'
      ? this.storage.getNounCountsByType()
      : new Uint32Array(0)
    const verbCounts = typeof this.storage.getVerbCountsByType === 'function'
      ? this.storage.getVerbCountsByType()
      : new Uint32Array(0)

    const entitiesByType: Record<string, number> = {}
    let entityCount = 0
    for (let i = 0; i < nounCounts.length; i++) {
      if (nounCounts[i] === 0) continue
      const name = NounTypeEnum[i as number]
      if (name) entitiesByType[name] = nounCounts[i]
      entityCount += nounCounts[i]
    }

    const relationsByType: Record<string, number> = {}
    let relationCount = 0
    for (let i = 0; i < verbCounts.length; i++) {
      if (verbCounts[i] === 0) continue
      const name = VerbTypeEnum[i as number]
      if (name) relationsByType[name] = verbCounts[i]
      relationCount += verbCounts[i]
    }

    const metadataStats = await this.metadataIndex.getStats()
    let fieldRegistry: string[] = []
    try {
      if (typeof this.metadataIndex.getFieldStatistics === 'function') {
        const fieldStats = await this.metadataIndex.getFieldStatistics()
        fieldRegistry = Array.from(fieldStats.keys()).sort()
      }
    } catch {
      // Field stats unavailable on this metadata index implementation — leave empty.
    }

    const writerLock = typeof this.storage.readWriterLock === 'function'
      ? await this.storage.readWriterLock()
      : null

    const storageBackend = this.storage.constructor.name
    // FileSystemStorage keeps its root directory in a `rootDir` member that
    // BaseStorage doesn't declare; surfaced opportunistically for operators.
    const rootDir = (this.storage as BaseStorage & { rootDir?: unknown }).rootDir

    return {
      mode: this.isReadOnly ? 'reader' : 'writer',
      entityCount,
      entitiesByType,
      relationCount,
      relationsByType,
      fieldRegistry,
      indexHealth: await (async () => {
        const graphSize = await this.graphIndex.size()
        return {
          vector: this.index.size() > 0 || entityCount === 0,
          metadata: metadataStats.totalEntries > 0 || entityCount === 0,
          graph: graphSize > 0 || relationCount === 0
        }
      })(),
      storage: {
        backend: storageBackend,
        rootDir: typeof rootDir === 'string' ? rootDir : undefined
      },
      writerLock: writerLock || undefined,
      version: getBrainyVersion()
    }
  }

  /**
   * Plugin and provider diagnostics — shows what's active and how subsystems are wired.
   *
   * @example
   * ```typescript
   * const diag = brain.diagnostics()
   * console.log(diag.providers) // { vector: { source: 'plugin' }, ... }
   * console.log(diag.indexes.graph.wiredToStorage) // true
   * ```
   */
  diagnostics(): DiagnosticsResult {
    const wellKnownKeys = [
      'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache',
      'vector', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
    ] as const

    const providers: Record<string, { source: 'plugin' | 'default' }> = {}
    for (const key of wellKnownKeys) {
      providers[key] = {
        source: this.pluginRegistry.hasProvider(key) ? 'plugin' : 'default'
      }
    }

    const hnswSize = this.index.size()
    const metadataInitialized = !!this.metadataIndex
    const graphInitialized = !!this.graphIndex
    // Boundary: identity-compares against BaseStorage's protected `graphIndex`
    // member to report whether the live graph index is the storage-wired one.
    // Diagnostics-only peek — the public accessor (`getGraphIndex()`) is async
    // and lazily *creates* the index, which would defeat the wiring check.
    const storageGraphIndex = (this.storage as unknown as {
      graphIndex?: GraphAdjacencyIndex
    }).graphIndex

    return {
      version: getBrainyVersion(),
      plugins: {
        active: this.pluginRegistry.getActivePlugins(),
        count: this.pluginRegistry.getActivePlugins().length
      },
      providers,
      indexes: {
        hnsw: {
          size: hnswSize,
          type: this.index.constructor.name
        },
        metadata: {
          type: this.metadataIndex?.constructor.name || 'none',
          initialized: metadataInitialized
        },
        graph: {
          type: this.graphIndex?.constructor.name || 'none',
          initialized: graphInitialized,
          wiredToStorage: graphInitialized && storageGraphIndex === this.graphIndex
        }
      }
    }
  }

  /**
   * Assert that specific providers are supplied by a plugin (not using JS fallback).
   *
   * Call after init() in production to fail fast if a paid plugin (e.g. cor)
   * isn't providing the expected acceleration. Throws if any listed key is using
   * the default JavaScript implementation.
   *
   * @param keys - Provider keys that MUST come from a plugin
   * @throws Error listing which providers are falling back to defaults
   *
   * @example
   * ```typescript
   * const brain = new Brainy()
   * await brain.init()
   *
   * // Fail fast if cor isn't providing these
   * brain.requireProviders(['distance', 'embeddings', 'metadataIndex', 'graphIndex'])
   * ```
   */
  requireProviders(keys: string[]): void {
    const missing = keys.filter(k => !this.pluginRegistry.hasProvider(k))
    if (missing.length > 0) {
      const active = this.pluginRegistry.getActivePlugins()
      const pluginInfo = active.length > 0
        ? `Active plugins: ${active.join(', ')}`
        : 'No plugins active'
      throw new Error(
        `[brainy] Required providers using JS fallback: ${missing.join(', ')}. ` +
        `${pluginInfo}. ` +
        `These providers must be supplied by a plugin for this deployment. ` +
        `Check plugin installation, license, and native module availability.`
      )
    }
  }

  /**
   * Efficient Pagination API - Production-scale pagination using index-first approach
   * Automatically optimizes based on query type and applies pagination at the index level
   */
  get pagination() {
    return {
      // Get paginated results with automatic optimization
      find: async (params: FindParams<T> & { page?: number, pageSize?: number }) => {
        const page = params.page || 1
        const pageSize = params.pageSize || 10
        const offset = (page - 1) * pageSize

        return this.find({
          ...params,
          limit: pageSize,
          offset
        })
      },

      // Get total count for pagination UI (O(1) when possible)
      count: async (params: Omit<FindParams<T>, 'limit' | 'offset'>) => {
        // Match-all normalization (shared with find()): an empty `where: {}`
        // carries no predicates. Counting it as a filter would route through
        // getIdsForFilter({}) → [] → a silent count of 0 while rows exist.
        const constrainingWhere = whereConstrains(params.where) ? params.where : undefined

        // For simple type queries, use O(1) index counting
        if (params.type && !params.subtype && !params.query && !constrainingWhere && !params.connected) {
          const types = Array.isArray(params.type) ? params.type : [params.type]
          return types.reduce((sum, type) => sum + this.metadataIndex.getEntityCountByType(type), 0)
        }

        // For complex queries, use metadata index for efficient counting
        if (constrainingWhere || params.subtype || params.service) {
          let filter: any = {}
          if (constrainingWhere) {
            // Where keys pass through UNTOUCHED — the one addressing law
            // parses them at the index boundary (bare = user metadata,
            // system.* = engine scalars). The old where.type→noun alias is
            // dead: a bare 'type' is the user's own field now.
            Object.assign(filter, constrainingWhere)
          }
          if (params.service) filter['system.service'] = params.service
          if (params.subtype !== undefined) {
            filter['system.subtype'] = Array.isArray(params.subtype)
              ? { oneOf: params.subtype }
              : params.subtype
          }
          if (params.type) {
            const types = Array.isArray(params.type) ? params.type : [params.type]
            if (types.length === 1) {
              filter['system.type'] = types[0]
            } else {
              const baseFilter = { ...filter }
              filter = {
                anyOf: types.map(type => ({ 'system.type': type, ...baseFilter }))
              }
            }
          }

          const filteredIds = await this.filterIdsBelted(filter)
          return filteredIds.length
        }

        // Fallback: total entity count
        return this.metadataIndex.getTotalEntityCount()
      },

      // Get pagination metadata
      meta: async (params: FindParams<T> & { page?: number, pageSize?: number }) => {
        const page = params.page || 1
        const pageSize = params.pageSize || 10
        const totalCount = await this.pagination.count(params)
        const totalPages = Math.ceil(totalCount / pageSize)

        return {
          page,
          pageSize,
          totalCount,
          totalPages,
          hasNext: page < totalPages,
          hasPrev: page > 1
        }
      }
    }
  }

  /**
   * Streaming API - Process millions of entities with constant memory using existing Pipeline
   * Integrates with index-based optimizations for maximum efficiency
   */
  get streaming(): {
    entities: (filter?: Partial<FindParams<T>>) => AsyncGenerator<Entity<T>>
    search: (params: FindParams<T>, batchSize?: number) => AsyncGenerator<{ id: string; score: number; entity: Entity<T> }>
    relationships: (filter?: { type?: string; sourceId?: string; targetId?: string }) => AsyncGenerator<any>
    pipeline: (source: AsyncIterable<any>) => any
    process: (processor: (entity: Entity<T>) => Promise<Entity<T>>, filter?: Partial<FindParams<T>>, options?: { batchSize: number; parallel: number }) => Promise<void>
  } {
    return {
      // Stream all entities with optional filtering
      entities: async function* (this: Brainy<T>, filter?: Partial<FindParams<T>>) {
        // Match-all normalization (shared with find()): an empty `where: {}`
        // carries no predicates — routing it through getIdsForFilter({})
        // would stream NOTHING while storage holds rows. Treat it as absent
        // so it falls to the unfiltered storage-paginated walk below.
        const constrainingWhere = whereConstrains(filter?.where) ? filter!.where : undefined
        if (filter && (filter.type || filter.subtype || constrainingWhere || filter.service)) {
          // Use MetadataIndexManager for efficient filtered streaming
          let filterObj: any = {}
          if (constrainingWhere) {
            // Where keys pass through — the addressing law parses them at
            // the index boundary; the type→noun alias is dead.
            Object.assign(filterObj, constrainingWhere)
          }
          if (filter.service) filterObj['system.service'] = filter.service
          if (filter.subtype !== undefined) {
            filterObj['system.subtype'] = Array.isArray(filter.subtype)
              ? { oneOf: filter.subtype }
              : filter.subtype
          }
          if (filter.type) {
            const types = Array.isArray(filter.type) ? filter.type : [filter.type]
            if (types.length === 1) {
              filterObj['system.type'] = types[0]
            } else {
              const baseFilterObj = { ...filterObj }
              filterObj = {
                anyOf: types.map(type => ({ 'system.type': type, ...baseFilterObj }))
              }
            }
          }

          const filteredIds = await this.filterIdsBelted(filterObj)

          // Stream filtered entities in batches for memory efficiency
          const batchSize = 100
          for (let i = 0; i < filteredIds.length; i += batchSize) {
            const batchIds = filteredIds.slice(i, i + batchSize)
            for (const id of batchIds) {
              const entity = await this.get(id)
              if (entity) yield entity as Entity<T>
            }
          }
        } else {
          // Stream all entities using storage adapter pagination
          let offset = 0
          const batchSize = 100
          let hasMore = true

          while (hasMore) {
            const result = await this.storage.getNouns({
              pagination: { offset, limit: batchSize }
            })

            for (const noun of result.items) {
              // Convert HNSWNoun to Entity<T>
              yield noun as unknown as Entity<T>
            }

            hasMore = result.hasMore
            offset += batchSize
          }
        }
      }.bind(this),

      // Stream search results efficiently
      search: async function* (this: Brainy<T>, params: FindParams<T>, batchSize = 50) {
        const originalLimit = params.limit
        let offset = 0
        let hasMore = true

        while (hasMore) {
          const batchResults = await this.find({
            ...params,
            limit: batchSize,
            offset
          })

          for (const result of batchResults) {
            yield result
          }

          hasMore = batchResults.length === batchSize
          offset += batchSize

          // Respect original limit if specified
          if (originalLimit && offset >= originalLimit) {
            break
          }
        }
      }.bind(this),

      // Stream relationships efficiently. Cursor-based when the adapter supports it
      // (resumes after the last verb — O(N) for a full walk) and falls back to offset
      // otherwise. Offset paging here re-scanned from the start every page (O(N²)).
      relationships: async function* (this: Brainy<T>, filter?: { type?: string, sourceId?: string, targetId?: string }) {
        let offset = 0
        let cursor: string | undefined
        const batchSize = 100

        for (;;) {
          const result = await this.storage.getVerbs({
            pagination: cursor ? { limit: batchSize, cursor } : { offset, limit: batchSize },
            filter
          })

          for (const verb of result.items) {
            yield verb
          }

          if (!result.hasMore || result.items.length === 0) break
          if (result.nextCursor) {
            cursor = result.nextCursor
          } else {
            offset += result.items.length
          }
        }
      }.bind(this),

      // Create processing pipeline from stream
      pipeline: (source: AsyncIterable<any>) => {
        return createPipeline(this).source(source)
      },

      // Batch process entities with Pipeline system
      process: async function (this: Brainy<T>,
        processor: (entity: Entity<T>) => Promise<Entity<T>>,
        filter?: Partial<FindParams<T>>,
        options = { batchSize: 50, parallel: 4 }
      ) {
        return createPipeline(this)
          .source(this.streaming.entities(filter))
          .batch(options.batchSize)
          .parallelSink(async (batch: Entity<T>[]) => {
            await Promise.all(batch.map(processor))
          }, options.parallel)
          .run()
      }.bind(this)
    }
  }

  /**
   * O(1) Count API - Production-scale counting using existing indexes
   * Works across all storage adapters (FileSystem, OPFS, S3, Memory)
   *
   * Phase 1b Enhancement: Type-aware methods with 99.2% memory reduction
   */
  get counts() {
    return {
      // O(1) total entity count
      entities: () => this.metadataIndex.getTotalEntityCount(),

      // O(1) total relationship count
      relationships: () => this.graphIndex.getTotalRelationshipCount(),

      // O(1) count by type (string-based, backward compatible)
      // Added optional excludeVFS using Roaring bitmap intersection
      byType: async (typeOrOptions?: string | { excludeVFS?: boolean }, options?: { excludeVFS?: boolean }) => {
        // Handle overloaded signature: byType(type), byType({ excludeVFS }), byType(type, { excludeVFS })
        let type: string | undefined
        let excludeVFS = false

        if (typeof typeOrOptions === 'string') {
          type = typeOrOptions
          excludeVFS = options?.excludeVFS ?? false
        } else if (typeOrOptions && typeof typeOrOptions === 'object') {
          excludeVFS = typeOrOptions.excludeVFS ?? false
        }

        if (excludeVFS) {
          const allCounts = this.metadataIndex.getAllEntityCounts()
          // Uses Roaring bitmap intersection - hardware accelerated
          const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts()

          if (type) {
            const total = allCounts.get(type) || 0
            const vfs = vfsCounts.get(type) || 0
            return total - vfs
          }

          // Return all counts with VFS subtracted
          const result: Record<string, number> = {}
          for (const [t, total] of allCounts) {
            const vfs = vfsCounts.get(t) || 0
            const nonVfs = total - vfs
            if (nonVfs > 0) {
              result[t] = nonVfs
            }
          }
          return result
        }

        // Default path (unchanged) - synchronous for backward compatibility
        if (type) {
          return this.metadataIndex.getEntityCountByType(type)
        }
        return Object.fromEntries(this.metadataIndex.getAllEntityCounts())
      },

      // Phase 1b: O(1) count by type enum (Uint32Array-based, more efficient)
      // Uses fixed-size type tracking: 676 bytes vs ~35KB with Maps (98.1% reduction)
      byTypeEnum: (type: NounType) => {
        return this.metadataIndex.getEntityCountByTypeEnum(type)
      },

      // Phase 1b: Get top N noun types by entity count (useful for cache warming)
      topTypes: (n: number = 10) => {
        return this.metadataIndex.getTopNounTypes(n)
      },

      /**
       * O(1) subtype counts for a given NounType.
       *
       * Returns the count for a single (type, subtype) pair when `subtype` is
       * passed; returns the full subtype → count map for that NounType when omitted.
       * Backed by the persisted `_system/subtype-statistics.json` rollup — no
       * scan, no storage round-trip.
       *
       * @param type - The NounType to count subtypes within
       * @param subtype - Optional specific subtype string for O(1) point count
       * @returns A number when `subtype` is given, otherwise a `Record<subtype, count>` (empty `{}` if none)
       *
       * @example Get all subtypes of Person
       * const counts = brain.counts.bySubtype(NounType.Person)
       * // → { employee: 56, customer: 847, vendor: 34 }
       *
       * @example O(1) point count
       * const employees = brain.counts.bySubtype(NounType.Person, 'employee')
       * // → 56
       */
      bySubtype: (type: NounType, subtype?: string): number | Record<string, number> => {
        const subtypeMap = typeof this.storage.getSubtypeCountsByType === 'function'
          ? this.storage.getSubtypeCountsByType()
          : null
        if (!subtypeMap) {
          return subtype !== undefined ? 0 : {}
        }
        const typeIdx = TypeUtils.getNounIndex(type)
        const inner = subtypeMap.get(typeIdx)
        if (!inner) {
          return subtype !== undefined ? 0 : {}
        }
        if (subtype !== undefined) {
          return inner.get(subtype) || 0
        }
        const result: Record<string, number> = {}
        for (const [k, v] of inner.entries()) result[k] = v
        return result
      },

      /**
       * Top N subtypes for a NounType, sorted by count (descending).
       *
       * @param type - The NounType to rank subtypes within
       * @param n - Maximum number of (subtype, count) pairs to return (default: 10)
       * @returns Array of `[subtype, count]` tuples, highest count first
       *
       * @example
       * const top3 = brain.counts.topSubtypes(NounType.Person, 3)
       * // → [['customer', 847], ['employee', 56], ['vendor', 34]]
       */
      topSubtypes: (type: NounType, n: number = 10): Array<[string, number]> => {
        const subtypeMap = typeof this.storage.getSubtypeCountsByType === 'function'
          ? this.storage.getSubtypeCountsByType()
          : null
        if (!subtypeMap) return []
        const typeIdx = TypeUtils.getNounIndex(type)
        const inner = subtypeMap.get(typeIdx)
        if (!inner) return []
        return Array.from(inner.entries())
          .sort((a, b) => b[1] - a[1])
          .slice(0, n)
      },

      // Phase 1b: Get top N verb types by count
      topVerbTypes: (n: number = 10) => {
        return this.metadataIndex.getTopVerbTypes(n)
      },

      // Phase 1b: Get all noun type counts as typed Map
      // More efficient than byType() for type-aware queries
      allNounTypeCounts: () => {
        return this.metadataIndex.getAllNounTypeCounts()
      },

      // Phase 1b: Get all verb type counts as typed Map
      allVerbTypeCounts: () => {
        return this.metadataIndex.getAllVerbTypeCounts()
      },

      // O(1) count by relationship type
      byRelationshipType: (type?: string) => {
        if (type) {
          return this.graphIndex.getRelationshipCountByType(type)
        }
        return Object.fromEntries(this.graphIndex.getAllRelationshipCounts())
      },

      /**
       * O(1) subtype counts for a given VerbType. Verb-side mirror of
       * `bySubtype`. Returns the count for a single (verb, subtype) pair when
       * `subtype` is passed; returns the full subtype → count map when omitted.
       * Backed by the persisted `_system/verb-subtype-statistics.json` rollup.
       *
       * @param verb - The VerbType to count subtypes within
       * @param subtype - Optional specific subtype string for O(1) point count
       *
       * @example
       * brain.counts.byRelationshipSubtype(VerbType.ReportsTo)
       * // → { direct: 12, 'dotted-line': 3 }
       *
       * brain.counts.byRelationshipSubtype(VerbType.ReportsTo, 'direct')
       * // → 12
       */
      byRelationshipSubtype: (verb: VerbType, subtype?: string): number | Record<string, number> => {
        const verbSubtypeMap = typeof this.storage.getVerbSubtypeCountsByType === 'function'
          ? this.storage.getVerbSubtypeCountsByType()
          : null
        if (!verbSubtypeMap) {
          return subtype !== undefined ? 0 : {}
        }
        const verbIdx = TypeUtils.getVerbIndex(verb)
        const inner = verbSubtypeMap.get(verbIdx)
        if (!inner) {
          return subtype !== undefined ? 0 : {}
        }
        if (subtype !== undefined) {
          return inner.get(subtype) || 0
        }
        const result: Record<string, number> = {}
        for (const [k, v] of inner.entries()) result[k] = v
        return result
      },

      /**
       * Top N subtypes for a VerbType, sorted by count (descending). Mirror of
       * `topSubtypes` for verbs.
       *
       * @example
       * brain.counts.topRelationshipSubtypes(VerbType.ReportsTo, 5)
       * // → [['direct', 12], ['dotted-line', 3]]
       */
      topRelationshipSubtypes: (verb: VerbType, n: number = 10): Array<[string, number]> => {
        const verbSubtypeMap = typeof this.storage.getVerbSubtypeCountsByType === 'function'
          ? this.storage.getVerbSubtypeCountsByType()
          : null
        if (!verbSubtypeMap) return []
        const verbIdx = TypeUtils.getVerbIndex(verb)
        const inner = verbSubtypeMap.get(verbIdx)
        if (!inner) return []
        return Array.from(inner.entries())
          .sort((a, b) => b[1] - a[1])
          .slice(0, n)
      },

      // O(1) count by field-value criteria
      byCriteria: async (field: string, value: any) => {
        return this.metadataIndex.getCountForCriteria(field, value)
      },

      /**
       * Counts by value for a field registered via `brain.trackField()`. Reads
       * from the backing `__fieldCounts__<name>` aggregate (Layer 2 of the
       * subtype-and-facets primitive). Backfill-on-define means the first call
       * scans existing entities, subsequent calls are O(groups).
       *
       * Without `options.type`: returns the cross-NounType total per value.
       * With `options.type`: returns per-value counts for that NounType only —
       * requires the field to have been registered with `perType: true`.
       *
       * @param name - The tracked field name
       * @param options.type - Optional NounType filter (requires perType registration)
       * @returns `{ value: count }` map. Empty when the field wasn't tracked
       *   (no aggregate exists) or no entities have set it yet.
       * @throws If `options.type` is passed but the field was not registered with `perType: true`
       *
       * @example
       * brain.trackField('status', { perType: true })
       * await brain.add({ data: 'Ship it', type: NounType.Task, metadata: { status: 'todo' } })
       * await brain.counts.byField('status')
       * // → { todo: 1 }
       * await brain.counts.byField('status', { type: NounType.Task })
       * // → { todo: 1 }
       */
      byField: async (
        name: string,
        options?: { type?: NounType }
      ): Promise<Record<string, number>> => {
        const tracked = this._trackedFields.get(name)
        if (!tracked) return {}
        if (options?.type !== undefined && !tracked.perType) {
          throw new Error(
            `counts.byField('${name}'): per-type breakdown requested but the field was registered without perType:true. Re-call trackField('${name}', { perType: true }).`
          )
        }
        const aggregateName = this.fieldCountsAggregateName(name)
        if (!this._aggregationIndex || !this._aggregationIndex.hasAggregate(aggregateName)) {
          return {}
        }
        const rows = await this.queryAggregate(aggregateName)
        const result: Record<string, number> = {}
        for (const row of rows) {
          const value = row.groupKey?.[name]
          // Skip the aggregation engine's "missing-value" sentinel: entities that
          // don't have the tracked field at all (e.g. the VFS root) bucket under
          // '__null__' and would otherwise pollute the count map.
          if (value === undefined || value === null || value === '__null__') continue
          if (options?.type !== undefined && row.groupKey?.['system.type'] !== options.type) continue
          const key = String(value)
          result[key] = (result[key] || 0) + (typeof row.metrics?.count === 'number' ? row.metrics.count : row.count)
        }
        return result
      },

      // Get all type counts as Map for performance-critical operations
      getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(),

      // Get complete statistics
      // Added optional excludeVFS using Roaring bitmap intersection
      getStats: async (options?: { excludeVFS?: boolean }) => {
        if (options?.excludeVFS) {
          const allCounts = this.metadataIndex.getAllEntityCounts()
          // Uses Roaring bitmap intersection - hardware accelerated
          const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts()

          // Compute non-VFS counts via subtraction
          const byType: Record<string, number> = {}
          let total = 0

          for (const [type, count] of allCounts) {
            const vfs = vfsCounts.get(type) || 0
            const nonVfs = count - vfs
            if (nonVfs > 0) {
              byType[type] = nonVfs
              total += nonVfs
            }
          }

          const entityStats = { total, byType }
          const relationshipStats = this.graphIndex.getRelationshipStats()

          return {
            entities: entityStats,
            relationships: relationshipStats,
            density: total > 0 ? relationshipStats.totalRelationships / total : 0
          }
        }

        // Default path (unchanged) - synchronous for backward compatibility
        const entityStats = {
          total: this.metadataIndex.getTotalEntityCount(),
          byType: Object.fromEntries(this.metadataIndex.getAllEntityCounts())
        }
        const relationshipStats = this.graphIndex.getRelationshipStats()

        return {
          entities: entityStats,
          relationships: relationshipStats,
          density: entityStats.total > 0 ? relationshipStats.totalRelationships / entityStats.total : 0
        }
      }
    }
  }

  /**
   * Get complete statistics - convenience method
   * For more granular counting, use brain.counts API
   * Added optional excludeVFS using Roaring bitmap intersection
   * @param options Optional settings - excludeVFS: filter out VFS entities
   * @returns Complete statistics including entities, relationships, and density
   */
  async getStats(options?: { excludeVFS?: boolean }) {
    return this.counts.getStats(options)
  }

  /**
   * Distinct subtypes seen for a given NounType.
   *
   * Reads from the subtype-statistics rollup — no scan, no storage round-trip.
   * The returned list is the vocabulary actually observed in the data, not a
   * registered schema (Brainy doesn't validate subtype vocabulary; that's a
   * consumer concern).
   *
   * @param type - The NounType to enumerate subtypes for
   * @returns Sorted list of distinct subtype strings (empty if none)
   *
   * @example
   * const personSubtypes = brain.subtypesOf(NounType.Person)
   * // → ['customer', 'employee', 'vendor']
   */
  subtypesOf(type: NounType): string[] {
    const subtypeMap = typeof this.storage.getSubtypeCountsByType === 'function'
      ? this.storage.getSubtypeCountsByType()
      : null
    if (!subtypeMap) return []
    const typeIdx = TypeUtils.getNounIndex(type)
    const inner = subtypeMap.get(typeIdx)
    if (!inner) return []
    return Array.from(inner.keys()).sort()
  }

  /**
   * Distinct subtypes seen for a given VerbType. Mirror of `subtypesOf` for
   * relationships. Reads from the verb subtype-statistics rollup — no scan, no
   * storage round-trip. Vocabulary is observed (what's actually in the data),
   * not registered.
   *
   * @param verb - The VerbType to enumerate subtypes for
   * @returns Sorted list of distinct subtype strings (empty if none)
   *
   * @example
   * const variants = brain.relationshipSubtypesOf(VerbType.ReportsTo)
   * // → ['direct', 'dotted-line']
   */
  relationshipSubtypesOf(verb: VerbType): string[] {
    const verbSubtypeMap = typeof this.storage.getVerbSubtypeCountsByType === 'function'
      ? this.storage.getVerbSubtypeCountsByType()
      : null
    if (!verbSubtypeMap) return []
    const verbIdx = TypeUtils.getVerbIndex(verb)
    const inner = verbSubtypeMap.get(verbIdx)
    if (!inner) return []
    return Array.from(inner.keys()).sort()
  }

  /**
   * Find entities and relationships missing a `subtype` value, grouped by type.
   *
   * The diagnostic pair to `fillSubtypes()` / `migrateField()` — answers the
   * question "what would strict subtype enforcement reject?". 8.0 makes
   * `requireSubtype: true` the default, so run this when opening a pre-8.0
   * brain (with `requireSubtype: false` as the temporary escape hatch), then
   * back-fill the reported gaps with `fillSubtypes(rules)`.
   *
   * Streams the brain via the same paginated `storage.getNouns()` /
   * `storage.getVerbs()` pattern `fillSubtypes()` uses — safe for large brains
   * but linear in `O(N)`. A native index provider may serve this from a
   * column-store null-subtype bitmap in the future for sub-linear performance
   * on billion-scale brains.
   *
   * @param options.includeVFS - When `false` (default), entities marked with
   *   `metadata.isVFSEntity` or `metadata.isVFS` are excluded from the report —
   *   these bypass enforcement anyway, so counting them is noise. Pass `true`
   *   to include them.
   * @param options.batchSize - Pagination batch size (default `200`).
   * @param options.onProgress - Optional progress callback invoked after each batch.
   * @returns Report with per-type counts of entities/relationships without a
   *   subtype, plus the overall total and a one-line recommendation pointing at
   *   `fillSubtypes()`.
   *
   * @example Find pre-existing gaps before turning on strict mode
   * const report = await brain.audit()
   * if (report.total > 0) {
   *   console.warn('Found ' + report.total + ' entities/edges without subtype:')
   *   console.warn(report.entitiesWithoutSubtype)
   *   console.warn(report.relationshipsWithoutSubtype)
   * }
   *
   * @since 7.30.1
   */
  async audit(options: {
    includeVFS?: boolean
    batchSize?: number
    onProgress?: (progress: { scanned: number; missingSubtype: number }) => void
  } = {}): Promise<{
    entitiesWithoutSubtype: Record<string, number>
    relationshipsWithoutSubtype: Record<string, number>
    total: number
    scanned: number
    recommendation: string
  }> {
    await this.ensureInitialized()

    const includeVFS = options.includeVFS === true
    const batchSize = Math.max(1, options.batchSize ?? 200)

    const entitiesWithoutSubtype: Record<string, number> = {}
    const relationshipsWithoutSubtype: Record<string, number> = {}
    let scanned = 0
    let missingSubtype = 0

    const reportProgress = (): void => {
      if (options.onProgress) options.onProgress({ scanned, missingSubtype })
    }

    // Scan nouns
    let offset = 0
    while (true) {
      const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } })
      if (page.items.length === 0) break
      for (const noun of page.items) {
        scanned++
        // Legacy stored shapes carried the entity type under `noun` — read both.
        const n: HNSWNounWithMetadata & { noun?: NounType } = noun
        // Skip VFS infrastructure entities unless includeVFS is set — they
        // bypass enforcement via the isVFSEntity marker anyway, so listing
        // them in the report would mislead consumers into thinking they have
        // a migration gap they actually don't.
        if (!includeVFS && (n.metadata?.isVFSEntity === true || n.metadata?.isVFS === true)) continue
        const subtype = typeof n.subtype === 'string' ? n.subtype : undefined
        if (!subtype || subtype.length === 0) {
          missingSubtype++
          const typeKey = String(n.type ?? n.noun ?? 'unknown')
          entitiesWithoutSubtype[typeKey] = (entitiesWithoutSubtype[typeKey] || 0) + 1
        }
      }
      reportProgress()
      if (!page.hasMore) break
      offset += page.items.length
    }

    // Scan verbs
    offset = 0
    while (true) {
      const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } })
      if (page.items.length === 0) break
      for (const verb of page.items) {
        scanned++
        // Legacy stored shapes carried the verb type under `type` — read both.
        const v: HNSWVerbWithMetadata & { type?: VerbType } = verb
        if (!includeVFS && (v.metadata?.isVFSEntity === true || v.metadata?.isVFS === true)) continue
        const subtype = typeof v.subtype === 'string' ? v.subtype : undefined
        if (!subtype || subtype.length === 0) {
          missingSubtype++
          const verbKey = String(v.verb ?? v.type ?? 'unknown')
          relationshipsWithoutSubtype[verbKey] = (relationshipsWithoutSubtype[verbKey] || 0) + 1
        }
      }
      reportProgress()
      if (!page.hasMore) break
      offset += page.items.length
    }

    const recommendation = missingSubtype === 0
      ? 'No subtype gaps detected — this brain is strict-mode-ready.'
      : 'Found ' + missingSubtype + ' entries without subtype. Back-fill with `brain.fillSubtypes(rules)` — one rule per NounType/VerbType, literal default or per-entry function. To lift an existing field into `subtype` instead, use `brain.migrateField({ from, to: \'subtype\' })`.'

    return {
      entitiesWithoutSubtype,
      relationshipsWithoutSubtype,
      total: missingSubtype,
      scanned,
      recommendation
    }
  }

  /**
   * Back-fill missing `subtype` values across the whole brain — the 8.0
   * migration helper for data written before subtype became required.
   *
   * 8.0 enforces a non-empty `subtype` on every write by default
   * (`requireSubtype: true`). A brain created on 7.x typically carries entities
   * and relationships without one; this method clears that debt in a single
   * idempotent pass so the opt-out (`requireSubtype: false`) can be removed.
   * The intended upgrade flow:
   *
   * 1. Open the brain with `requireSubtype: false` (temporary escape hatch).
   * 2. `await brain.audit()` — see what's missing, grouped by type.
   * 3. `await brain.fillSubtypes(rules)` — back-fill with one rule per type.
   * 4. Re-run `audit()` until `total === 0`, then drop the opt-out.
   *
   * **Rules.** One rule per NounType (entities) and/or VerbType
   * (relationships) — the two vocabularies don't overlap, so a single map
   * covers both sides. A rule is either a literal subtype string (blanket
   * default) or a function deriving the subtype from the entry; functions
   * subsume `where`-style filtering by returning `undefined` for entries they
   * decline (those stay untouched and count as `skipped`, so a later run with
   * a stricter rule can pick them up).
   *
   * **What is never touched:** entries that already carry a non-empty
   * `subtype` (re-running is a no-op on them), and — unless
   * `includeVFS: true` — Brainy's own VFS infrastructure entries
   * (`metadata.isVFSEntity` / `metadata.isVFS`), which bypass enforcement
   * anyway and are not migration debt.
   *
   * **Write strategy (deliberate):** each fill goes through the public
   * `update()` / `updateRelation()` paths, so every write is individually
   * atomic (storage record + indexes + subtype rollups commit together) and
   * bumps the entry's `_rev` like any other update. The pass is *not* one
   * whole-brain transaction: a migration over millions of entries inside a
   * single transaction would hold an unbounded working set and turn one bad
   * entry into an all-or-nothing failure. Idempotence is the recovery model —
   * a crashed or partially-failed run is resumed safely by re-running, because
   * only entries still missing a subtype are written.
   *
   * Streams via the same paginated `storage.getNouns()` / `storage.getVerbs()`
   * walk `audit()` uses — `O(N)` but constant memory. The noun pass is skipped
   * entirely when the map has no NounType rules, and vice versa.
   *
   * @param rules - Map of NounType/VerbType → literal subtype or rule function.
   *   Must contain at least one valid type key; invalid keys, empty-string
   *   literals, and non-string/non-function values throw before any data is
   *   touched.
   * @param options.includeVFS - Also fill VFS infrastructure entries (default
   *   `false` — they bypass enforcement and don't need a subtype).
   * @param options.batchSize - Pagination batch size (default `200`).
   * @param options.onProgress - Optional callback invoked after each batch.
   * @returns `{ scanned, filled, skipped, errors, byType }` — see
   *   {@link FillSubtypesResult}. After a clean run, `skipped` equals the
   *   remaining `audit().total`.
   * @throws If the brain is read-only, the rule map is empty/malformed, or a
   *   key is not a valid NounType/VerbType. Per-entry write failures do NOT
   *   throw — they are collected in `errors` and the pass continues.
   *
   * @example Back-fill entities and relationships in one pass
   * const report = await brain.fillSubtypes({
   *   [NounType.Person]: (e) => e.metadata?.kind ?? 'unspecified',
   *   [NounType.Document]: 'general',
   *   [VerbType.RelatedTo]: 'unspecified'
   * })
   * // → { scanned: 5200, filled: 1429, skipped: 0, errors: [], byType: { person: 800, document: 600, relatedTo: 29 } }
   *
   * @example Selective fill — decline entries a rule can't classify
   * await brain.fillSubtypes({
   *   [NounType.Person]: (e) => e.metadata?.department ? 'employee' : undefined
   * })
   * // Persons without a department stay untouched (counted as skipped).
   *
   * @since 8.0.0
   */
  async fillSubtypes(
    rules: FillSubtypeRules<T>,
    options: {
      includeVFS?: boolean
      batchSize?: number
      onProgress?: (progress: { scanned: number; filled: number; skipped: number }) => void
    } = {}
  ): Promise<FillSubtypesResult> {
    this.assertWritable('fillSubtypes')
    await this.ensureInitialized()

    // Validate the rule map up front — fail fast on shape errors before any
    // data is touched.
    if (!rules || typeof rules !== 'object' || Array.isArray(rules)) {
      throw new Error(
        'fillSubtypes: rules must be a map of NounType/VerbType → subtype string or rule function'
      )
    }
    const nounTypeValues = new Set<string>(Object.values(NounType))
    const verbTypeValues = new Set<string>(Object.values(VerbType))
    const nounRules = new Map<string, FillSubtypeRule<Entity<T>>>()
    const verbRules = new Map<string, FillSubtypeRule<Relation<T>>>()
    for (const [key, rule] of Object.entries(rules)) {
      if (rule === undefined) continue
      if (typeof rule !== 'string' && typeof rule !== 'function') {
        throw new Error(
          `fillSubtypes: rule for '${key}' must be a subtype string or a function (got ${typeof rule})`
        )
      }
      if (typeof rule === 'string' && rule.length === 0) {
        throw new Error(
          `fillSubtypes: rule for '${key}' is an empty string — a subtype must be non-empty`
        )
      }
      if (nounTypeValues.has(key)) {
        nounRules.set(key, rule as FillSubtypeRule<Entity<T>>)
      } else if (verbTypeValues.has(key)) {
        verbRules.set(key, rule as FillSubtypeRule<Relation<T>>)
      } else {
        throw new Error(`fillSubtypes: '${key}' is not a valid NounType or VerbType`)
      }
    }
    if (nounRules.size === 0 && verbRules.size === 0) {
      throw new Error(
        'fillSubtypes: rules map is empty — provide at least one NounType or VerbType rule'
      )
    }

    const includeVFS = options.includeVFS === true
    const batchSize = Math.max(1, options.batchSize ?? 200)

    let scanned = 0
    let filled = 0
    let skipped = 0
    const errors: Array<{ id: string; error: string }> = []
    const byType: Record<string, number> = {}

    const reportProgress = (): void => {
      if (options.onProgress) options.onProgress({ scanned, filled, skipped })
    }

    // Normalize a rule's output: only a non-empty string is a fill; anything
    // else (undefined, empty string) is a decline. An empty subtype would fail
    // the very strict-mode check this helper exists to satisfy.
    const normalize = (value: string | undefined): string | undefined =>
      typeof value === 'string' && value.length > 0 ? value : undefined

    // Pass 1: entities (skipped entirely when the map has no NounType rules).
    if (nounRules.size > 0) {
      let offset = 0
      while (true) {
        const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } })
        if (page.items.length === 0) break
        for (const noun of page.items) {
          scanned++
          // VFS infrastructure entries bypass enforcement via their marker, so
          // they're not migration debt — leave them alone unless asked.
          if (
            !includeVFS &&
            (noun.metadata?.isVFSEntity === true || noun.metadata?.isVFS === true)
          ) {
            continue
          }
          if (typeof noun.subtype === 'string' && noun.subtype.length > 0) continue // already filled — never overwrite
          const rule = nounRules.get(noun.type)
          if (rule === undefined) {
            skipped++
            continue
          }
          try {
            let subtype: string | undefined
            if (typeof rule === 'function') {
              const entity = await this.convertNounToEntity(noun)
              subtype = normalize(rule(entity))
            } else {
              subtype = rule
            }
            if (subtype === undefined) {
              skipped++
              continue
            }
            await this.update({ id: noun.id, subtype })
            filled++
            byType[noun.type] = (byType[noun.type] || 0) + 1
          } catch (err) {
            errors.push({
              id: noun.id,
              error: err instanceof Error ? err.message : String(err)
            })
          }
        }
        reportProgress()
        if (!page.hasMore) break
        offset += page.items.length
      }
    }

    // Pass 2: relationships (skipped entirely when the map has no VerbType rules).
    if (verbRules.size > 0) {
      let offset = 0
      while (true) {
        const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } })
        if (page.items.length === 0) break
        for (const verb of page.items) {
          scanned++
          if (
            !includeVFS &&
            (verb.metadata?.isVFSEntity === true || verb.metadata?.isVFS === true)
          ) {
            continue
          }
          if (typeof verb.subtype === 'string' && verb.subtype.length > 0) continue // already filled — never overwrite
          const rule = verbRules.get(verb.verb)
          if (rule === undefined) {
            skipped++
            continue
          }
          try {
            let subtype: string | undefined
            if (typeof rule === 'function') {
              // Project the stored verb onto the public Relation<T> shape the
              // rule function is typed against.
              const relation: Relation<T> = {
                id: verb.id,
                from: verb.sourceId,
                to: verb.targetId,
                type: verb.verb,
                weight: verb.weight,
                data: verb.data,
                metadata: verb.metadata as T,
                service: verb.service,
                createdAt: verb.createdAt,
                updatedAt: verb.updatedAt,
                confidence: verb.confidence
              }
              subtype = normalize(rule(relation))
            } else {
              subtype = rule
            }
            if (subtype === undefined) {
              skipped++
              continue
            }
            await this.updateRelation({ id: verb.id, subtype })
            filled++
            byType[verb.verb] = (byType[verb.verb] || 0) + 1
          } catch (err) {
            errors.push({
              id: verb.id,
              error: err instanceof Error ? err.message : String(err)
            })
          }
        }
        reportProgress()
        if (!page.hasMore) break
        offset += page.items.length
      }
    }

    return { scanned, filled, skipped, errors, byType }
  }

  /**
   * Stream-and-rewrite a field across every entity in the brain.
   *
   * Layer 3 of the subtype-and-facets primitive. Reads the value at `from`,
   * writes it to `to`, and (unless `readBoth: true`) clears the source. Use this
   * when migrating between field-name conventions or moving a value from
   * `metadata.*` / `data.*` up to a top-level standard field like `subtype`.
   *
   * **Path forms:**
   * - `'subtype'`, `'type'`, `'confidence'`, etc. — top-level standard fields
   *   (whatever appears in `STANDARD_ENTITY_FIELDS`)
   * - `'metadata.X'` — a key under `entity.metadata`
   * - `'data.X'` — a key under `entity.data` (when `data` is an object)
   * - `'X'` (bare, not standard) — shorthand for `metadata.X`
   *
   * **Behavior:**
   * - Streams entities in batches and rewrites in-place via `brain.update()`.
   *   Aggregations and indexes refresh automatically.
   * - Entities where the source path is absent or where the target already
   *   holds the same value are skipped (idempotent — safe to re-run).
   * - With `readBoth: true`, the source value is preserved alongside the new
   *   target so legacy consumers can keep querying the old path during a
   *   deprecation window. Re-run with `readBoth: false` when ready to clear.
   *
   * @param options.from - Source path
   * @param options.to - Destination path
   * @param options.readBoth - If true, leave the source value in place (default false → source cleared)
   * @param options.batchSize - Entities per batch (default 100)
   * @param options.onProgress - Optional callback invoked after each batch
   * @returns Migration summary — counts and per-entity errors
   *
   * @example One-shot migration
   * await brain.migrateField({ from: 'metadata.kind', to: 'subtype' })
   * // → { scanned: 1500, migrated: 1500, skipped: 0, errors: [] }
   *
   * @example Deprecation window — keep both fields readable
   * await brain.migrateField({ from: 'data.kind', to: 'subtype', readBoth: true })
   * // ...consumers switch reads to subtype over time...
   * await brain.migrateField({ from: 'data.kind', to: 'subtype' })
   * // ...source cleared in the final sweep.
   */
  async migrateField(options: {
    from: string
    to: string
    readBoth?: boolean
    batchSize?: number
    onProgress?: (progress: { scanned: number; migrated: number }) => void
    /**
     * Which entity kind to walk. Defaults to `'noun'` for backward compat with
     * 7.29.0. Use `'verb'` to migrate relationship fields, or `'both'` to walk
     * nouns then verbs in one pass. Path forms (`'subtype'`, `'metadata.X'`,
     * `'data.X'`) are identical on both sides.
     */
    entityKind?: 'noun' | 'verb' | 'both'
  }): Promise<{
    scanned: number
    migrated: number
    skipped: number
    errors: Array<{ id: string; error: string }>
  }> {
    this.assertWritable('migrateField')
    await this.ensureInitialized()

    if (!options.from || typeof options.from !== 'string') {
      throw new Error('migrateField: `from` must be a non-empty string path')
    }
    if (!options.to || typeof options.to !== 'string') {
      throw new Error('migrateField: `to` must be a non-empty string path')
    }
    if (options.from === options.to) {
      throw new Error('migrateField: `from` and `to` are identical — nothing to do')
    }

    const fromPath = this.parseMigrationPath(options.from)
    const toPath = this.parseMigrationPath(options.to)
    const batchSize = Math.max(1, options.batchSize ?? 100)
    const readBoth = options.readBoth === true
    const entityKind = options.entityKind ?? 'noun'

    let scanned = 0
    let migrated = 0
    let skipped = 0
    const errors: Array<{ id: string; error: string }> = []

    const reportProgress = (): void => {
      if (options.onProgress) {
        options.onProgress({ scanned, migrated })
      }
    }

    // Walk nouns when entityKind is 'noun' or 'both'.
    if (entityKind === 'noun' || entityKind === 'both') {
      let offset = 0
      while (true) {
        const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } })
        if (page.items.length === 0) break
        for (const noun of page.items) {
          scanned++
          try {
            const entity = await this.convertNounToEntity(noun)
            const sourceValue = this.readPath(entity, fromPath)
            if (sourceValue === undefined || sourceValue === null) {
              skipped++
              continue
            }
            const targetValue = this.readPath(entity, toPath)
            if (targetValue === sourceValue && (readBoth || !this.pathExists(entity, fromPath))) {
              skipped++
              continue
            }
            const update = this.buildMigrationUpdate(entity, fromPath, toPath, sourceValue, readBoth)
            await this.update(update)
            migrated++
          } catch (err) {
            errors.push({
              id: noun.id ?? '<unknown>',
              error: err instanceof Error ? err.message : String(err)
            })
          }
        }
        reportProgress()
        if (!page.hasMore) break
        offset += page.items.length
      }
    }

    // Walk verbs when entityKind is 'verb' or 'both'. Mirror of the noun loop —
    // the path forms (`'subtype'`, `'metadata.X'`, `'data.X'`) and the
    // readPath / buildMigrationUpdate helpers all work for verbs because
    // `Relation<T>` carries the same shape (top-level standard fields +
    // metadata bag + optional data object). updateRelation() is the verb-side
    // mutator.
    if (entityKind === 'verb' || entityKind === 'both') {
      let offset = 0
      while (true) {
        const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } })
        if (page.items.length === 0) break
        for (const verb of page.items) {
          scanned++
          try {
            const edge = this.verbToRelationLike(verb)
            const sourceValue = this.readPath(edge, fromPath)
            if (sourceValue === undefined || sourceValue === null) {
              skipped++
              continue
            }
            const targetValue = this.readPath(edge, toPath)
            if (targetValue === sourceValue && (readBoth || !this.pathExists(edge, fromPath))) {
              skipped++
              continue
            }
            const update = this.buildRelationMigrationUpdate(edge, fromPath, toPath, sourceValue, readBoth)
            await this.updateRelation(update)
            migrated++
          } catch (err) {
            errors.push({
              id: verb.id ?? '<unknown>',
              error: err instanceof Error ? err.message : String(err)
            })
          }
        }
        reportProgress()
        if (!page.hasMore) break
        offset += page.items.length
      }
    }

    return { scanned, migrated, skipped, errors }
  }

  /**
   * Project a storage verb shape onto the Entity<T>-shaped surface that
   * `readPath` / `pathExists` already understand. The migration helpers were
   * written for nouns; making them work for verbs is just a shape projection —
   * `subtype` / `metadata` / `data` live at the same paths on both sides.
   */
  private verbToRelationLike(verb: any): Entity<T> {
    return {
      id: verb.id,
      vector: verb.vector,
      // Deliberate projection: the edge's VerbType occupies Entity.type so the
      // noun-oriented path helpers (readPath/pathExists) work unchanged.
      type: (verb.verb ?? verb.type) as NounType,
      subtype: verb.subtype,
      data: verb.data,
      metadata: verb.metadata as T,
      service: verb.service,
      createdAt: verb.createdAt,
      updatedAt: verb.updatedAt,
      createdBy: verb.createdBy,
      confidence: verb.confidence,
      weight: verb.weight
    }
  }

  /**
   * Build an `UpdateRelationParams` payload that mirrors `buildMigrationUpdate`
   * for verbs. Top-level path → top-level on update; metadata path → metadata
   * bag with merge:false; data path → data bag.
   */
  private buildRelationMigrationUpdate(
    edge: Entity<T>,
    from: { kind: 'top' | 'metadata' | 'data'; field: string },
    to: { kind: 'top' | 'metadata' | 'data'; field: string },
    value: unknown,
    readBoth: boolean
  ): UpdateRelationParams<T> {
    const id = edge.id
    // Record intersection: `to.kind === 'top'` writes a dynamically-named
    // standard field (subtype/weight/confidence/...) onto the update payload.
    const update: UpdateRelationParams<T> & Record<string, unknown> = { id }

    if (to.kind === 'top') {
      update[to.field] = value
    } else if (to.kind === 'metadata') {
      const base = (edge.metadata as unknown as Record<string, unknown>) ?? {}
      const nextMeta: Record<string, unknown> = { ...base, [to.field]: value }
      if (!readBoth && from.kind === 'metadata') {
        delete nextMeta[from.field]
      }
      update.metadata = nextMeta as UpdateRelationParams<T>['metadata']
      update.merge = false
    } else {
      const base = (edge.data && typeof edge.data === 'object') ? { ...(edge.data as Record<string, unknown>) } : {}
      base[to.field] = value
      if (!readBoth && from.kind === 'data') {
        delete base[from.field]
      }
      update.data = base
    }

    if (!readBoth) {
      if (from.kind === 'metadata' && to.kind !== 'metadata') {
        const base = (edge.metadata as unknown as Record<string, unknown>) ?? {}
        const nextMeta: Record<string, unknown> = { ...base }
        delete nextMeta[from.field]
        update.metadata = nextMeta as UpdateRelationParams<T>['metadata']
        update.merge = false
      } else if (from.kind === 'data' && to.kind !== 'data') {
        const base = (edge.data && typeof edge.data === 'object') ? { ...(edge.data as Record<string, unknown>) } : null
        if (base) {
          delete base[from.field]
          update.data = base
        }
      } else if (from.kind === 'top' && from.field === 'subtype') {
        update.subtype = undefined
      }
    }

    return update
  }

  /**
   * Parse a dotted path used by `migrateField` into its routing kind + field name.
   * `'subtype'` → top-level standard; `'metadata.X'` → metadata; `'data.X'` → data;
   * bare non-standard names → metadata (matching `resolveEntityField`'s fallback).
   */
  private parseMigrationPath(path: string): { kind: 'top' | 'metadata' | 'data'; field: string } {
    const dotIdx = path.indexOf('.')
    if (dotIdx === -1) {
      if (STANDARD_ENTITY_FIELDS.has(path)) return { kind: 'top', field: path }
      return { kind: 'metadata', field: path }
    }
    const head = path.slice(0, dotIdx)
    const tail = path.slice(dotIdx + 1)
    if (!tail) throw new Error(`migrateField: invalid path '${path}' (trailing dot)`)
    if (head === 'metadata') return { kind: 'metadata', field: tail }
    if (head === 'data') return { kind: 'data', field: tail }
    throw new Error(
      `migrateField: unsupported path prefix '${head}'. Supported: 'metadata.X', 'data.X', or a bare field name.`
    )
  }

  /** Read the value at a parsed migration path. Returns `undefined` if absent. */
  private readPath(entity: Entity<T>, path: { kind: 'top' | 'metadata' | 'data'; field: string }): unknown {
    if (path.kind === 'top') return (entity as Entity<T> & Record<string, unknown>)[path.field]
    if (path.kind === 'metadata') {
      const bag = entity.metadata as unknown as Record<string, unknown> | undefined
      return bag?.[path.field]
    }
    const data = entity.data
    if (data && typeof data === 'object') {
      return (data as Record<string, unknown>)[path.field]
    }
    return undefined
  }

  /** Whether a parsed path resolves to a defined (non-undefined) value. */
  private pathExists(entity: Entity<T>, path: { kind: 'top' | 'metadata' | 'data'; field: string }): boolean {
    return this.readPath(entity, path) !== undefined
  }

  /**
   * Build an `UpdateParams` payload that copies the value from the source path
   * to the destination, and (unless `readBoth`) clears the source. Uses
   * `merge: false` on the metadata bag when clearing so we can omit the source
   * key authoritatively instead of relying on `undefined` round-trip behavior.
   */
  private buildMigrationUpdate(
    entity: Entity<T>,
    from: { kind: 'top' | 'metadata' | 'data'; field: string },
    to: { kind: 'top' | 'metadata' | 'data'; field: string },
    value: unknown,
    readBoth: boolean
  ): UpdateParams<T> {
    const id = entity.id
    // Record intersection: `to.kind === 'top'` writes a dynamically-named
    // standard field (subtype/weight/confidence/...) onto the update payload.
    const update: UpdateParams<T> & Record<string, unknown> = { id }

    // Apply destination write.
    if (to.kind === 'top') {
      update[to.field] = value
    } else if (to.kind === 'metadata') {
      const base = (entity.metadata as unknown as Record<string, unknown>) ?? {}
      const nextMeta: Record<string, unknown> = { ...base, [to.field]: value }
      if (!readBoth && from.kind === 'metadata') {
        delete nextMeta[from.field]
      }
      update.metadata = nextMeta as UpdateParams<T>['metadata']
      update.merge = false
    } else {
      // data path — only meaningful when data is an object
      const base = (entity.data && typeof entity.data === 'object') ? { ...(entity.data as Record<string, unknown>) } : {}
      base[to.field] = value
      if (!readBoth && from.kind === 'data') {
        delete base[from.field]
      }
      update.data = base
    }

    // Apply source clear if not already handled by the destination bag write.
    if (!readBoth) {
      if (from.kind === 'metadata' && to.kind !== 'metadata') {
        const base = (entity.metadata as unknown as Record<string, unknown>) ?? {}
        const nextMeta: Record<string, unknown> = { ...base }
        delete nextMeta[from.field]
        update.metadata = nextMeta as UpdateParams<T>['metadata']
        update.merge = false
      } else if (from.kind === 'data' && to.kind !== 'data') {
        const base = (entity.data && typeof entity.data === 'object') ? { ...(entity.data as Record<string, unknown>) } : null
        if (base) {
          delete base[from.field]
          update.data = base
        }
      } else if (from.kind === 'top' && (from.field === 'subtype')) {
        // Only subtype currently supports clearing at the top level via UpdateParams.
        // Other standard fields aren't user-mutable through this path.
        update.subtype = undefined
      }
    }

    return update
  }

  // ============= NEW EMBEDDING & ANALYSIS APIs =============

  /**
   * Batch embed multiple texts at once
   *
   * More efficient than calling embed() multiple times due to
   * WASM batch processing optimizations.
   *
   * @param texts Array of texts to embed
   * @returns Array of embedding vectors (384 dimensions each)
   *
   * @example
   * const embeddings = await brain.embedBatch([
   *   'Machine learning is fascinating',
   *   'Deep neural networks',
   *   'Natural language processing'
   * ])
   * // embeddings.length === 3
   * // embeddings[0].length === 384
   */
  async embedBatch(texts: string[], options?: { signal?: AbortSignal }): Promise<number[][]> {
    await this.ensureInitialized()

    if (texts.length === 0) {
      return []
    }

    // Plugin provides native batch embedding — single forward pass for all texts
    const batchProvider = this.pluginRegistry.getProvider<(texts: string[]) => Promise<number[][]>>('embedBatch')
    if (batchProvider) {
      return batchProvider(texts)
    }
    // Plugin provides single-text embedding engine — map through it
    if (this.pluginRegistry.hasProvider('embeddings')) {
      return Promise.all(texts.map(t => this.embedder(t)))
    }
    // Default: WASM batch API (single forward pass, more efficient than N calls)
    return await embeddingManager.embedBatch(texts, options)
  }

  /**
   * Calculate semantic similarity between two texts
   *
   * Returns a score from 0 (completely different) to 1 (identical meaning).
   * Uses cosine similarity on embedding vectors.
   *
   * @param textA First text
   * @param textB Second text
   * @returns Similarity score between 0 and 1
   *
   * @example
   * const score = await brain.similarity(
   *   'The cat sat on the mat',
   *   'A feline was resting on the rug'
   * )
   * // score ≈ 0.85 (high semantic similarity)
   */
  async similarity(textA: string, textB: string): Promise<number> {
    await this.ensureInitialized()

    // Embed both texts
    const [vectorA, vectorB] = await Promise.all([
      this.embedder(textA),
      this.embedder(textB)
    ])

    // Calculate cosine similarity (convert from distance)
    // cosineDistance returns 1 - similarity, so similarity = 1 - distance
    const distance = this.distance(vectorA, vectorB)
    return 1 - distance
  }

  /**
   * Zero-config hybrid highlighting
   *
   * Returns both exact text matches AND semantically similar concepts.
   * Perfect for UI highlighting at different levels:
   * - matchType: 'text' = exact word match (highlight strongly)
   * - matchType: 'semantic' = concept match (highlight softly)
   *
   * @param params.query - The search query
   * @param params.text - The text to highlight (e.g., entity.data)
   * @param params.granularity - 'word' | 'phrase' | 'sentence' (default: 'word')
   * @param params.threshold - Minimum similarity for semantic matches (default: 0.5)
   * @returns Array of highlights with text, score, position, and matchType
   *
   * @example
   * ```typescript
   * const highlights = await brain.highlight({
   *   query: "david the warrior",
   *   text: "David Smith is a brave fighter who battles dragons"
   * })
   * // Returns: [
   * //   { text: "David", score: 1.0, position: [0, 5], matchType: 'text' },       // Exact
   * //   { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, // Concept
   * //   { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' }  // Concept
   * // ]
   * ```
   */
  async highlight(params: import('./types/brainy.types.js').HighlightParams): Promise<import('./types/brainy.types.js').Highlight[]> {
    await this.ensureInitialized()

    const { query, text, granularity = 'word', threshold = 0.5, contentType, contentExtractor } = params

    if (!query || !text) {
      return []
    }

    // Extract text from structured content (JSON, HTML, Markdown)
    // Custom extractor takes priority, then built-in detection
    type ChunkWithCategory = { text: string, position: [number, number], contentCategory?: import('./types/brainy.types.js').ContentCategory }

    let segments: import('./types/brainy.types.js').ExtractedSegment[]
    if (contentExtractor) {
      segments = contentExtractor(text)
    } else {
      segments = extractForHighlighting(text, contentType)
    }

    // Build concatenated text from segments for position tracking
    // and split each segment into chunks based on granularity
    const allChunks: ChunkWithCategory[] = []
    let offset = 0
    for (const segment of segments) {
      const segmentChunks = this.splitForHighlighting(segment.text, granularity)
      for (const chunk of segmentChunks) {
        allChunks.push({
          text: chunk.text,
          position: [chunk.position[0] + offset, chunk.position[1] + offset],
          contentCategory: segment.contentCategory
        })
      }
      offset += segment.text.length + 1 // +1 for space between segments
    }

    if (allChunks.length === 0) {
      return []
    }

    // Production safety: Limit chunks to prevent memory explosion
    // At 500 words × 384 dimensions × 4 bytes = 768KB temp memory (acceptable)
    const MAX_HIGHLIGHT_CHUNKS = 500
    const chunks = allChunks.slice(0, MAX_HIGHLIGHT_CHUNKS)

    // Track all highlights (keyed by position to avoid duplicates)
    const highlightMap = new Map<string, import('./types/brainy.types.js').Highlight>()

    // === PHASE 1: Find exact text matches (score = 1.0, matchType = 'text') ===
    const queryWords = this.metadataIndex.tokenize(query)
    const queryWordsLower = new Set(queryWords.map(w => w.toLowerCase()))

    for (const chunk of chunks) {
      const chunkLower = chunk.text.toLowerCase().replace(/[^\w\s]/g, '')
      if (queryWordsLower.has(chunkLower)) {
        const key = `${chunk.position[0]}-${chunk.position[1]}`
        highlightMap.set(key, {
          text: chunk.text,
          score: 1.0,
          position: chunk.position,
          matchType: 'text',
          contentCategory: chunk.contentCategory
        })
      }
    }

    // === PHASE 2: Find semantic matches with timeout fallback ===
    // AbortController ensures the background semantic work (WASM batch embedding)
    // is cancelled on timeout or error, preventing event loop saturation and
    // WASM engine crashes from abandoned promises.
    const SEMANTIC_TIMEOUT_MS = 10_000
    const abortController = new AbortController()

    try {
      const semanticResult = await Promise.race([
        this.highlightSemanticPhase(query, chunks, threshold, highlightMap, abortController.signal),
        new Promise<'timeout'>((resolve) => setTimeout(() => resolve('timeout'), SEMANTIC_TIMEOUT_MS))
      ])

      if (semanticResult === 'timeout') {
        abortController.abort()
        const textHighlights = Array.from(highlightMap.values())
        return textHighlights.sort((a, b) => b.score - a.score)
      }
    } catch {
      abortController.abort()
      const textHighlights = Array.from(highlightMap.values())
      return textHighlights.sort((a, b) => b.score - a.score)
    }

    // Sort by score descending (text matches will be first with score=1.0)
    const highlights = Array.from(highlightMap.values())
    return highlights.sort((a, b) => b.score - a.score)
  }

  /**
   * Phase 2 of highlight(): semantic matching with batch embedding
   * @internal
   */
  private async highlightSemanticPhase(
    query: string,
    chunks: Array<{ text: string, position: [number, number], contentCategory?: import('./types/brainy.types.js').ContentCategory }>,
    threshold: number,
    highlightMap: Map<string, import('./types/brainy.types.js').Highlight>,
    signal?: AbortSignal
  ): Promise<void> {
    if (signal?.aborted) return

    // Get query embedding
    const queryVector = await this.embed(query)
    if (signal?.aborted) return

    // Batch embed all chunks using native WASM batch API
    const chunkTexts = chunks.map(c => c.text)
    const chunkVectors = await this.embedBatch(chunkTexts, { signal })
    if (signal?.aborted) return

    // Calculate semantic similarities
    for (let i = 0; i < chunks.length; i++) {
      const key = `${chunks[i].position[0]}-${chunks[i].position[1]}`

      // Skip if already a text match (text matches take priority)
      if (highlightMap.has(key)) continue

      const distance = this.distance(queryVector, chunkVectors[i])
      const similarity = 1 - distance

      if (similarity >= threshold) {
        highlightMap.set(key, {
          text: chunks[i].text,
          score: similarity,
          position: chunks[i].position,
          matchType: 'semantic',
          contentCategory: chunks[i].contentCategory
        })
      }
    }
  }

  /**
   * Split text into chunks for highlighting
   * @internal
   */
  private splitForHighlighting(text: string, granularity: string): Array<{ text: string, position: [number, number] }> {
    const results: Array<{ text: string, position: [number, number] }> = []

    if (granularity === 'word') {
      // Split on whitespace, track positions
      const regex = /\S+/g
      let match
      while ((match = regex.exec(text)) !== null) {
        // Skip stopwords
        if (!STOPWORDS.has(match[0].toLowerCase())) {
          results.push({ text: match[0], position: [match.index, match.index + match[0].length] })
        }
      }
    } else if (granularity === 'sentence') {
      // Split on sentence boundaries
      const regex = /[^.!?]+[.!?]+/g
      let match
      while ((match = regex.exec(text)) !== null) {
        results.push({ text: match[0].trim(), position: [match.index, match.index + match[0].length] })
      }
      // Handle text without sentence-ending punctuation
      if (results.length === 0 && text.trim()) {
        results.push({ text: text.trim(), position: [0, text.length] })
      }
    } else if (granularity === 'phrase') {
      // Sliding window of 2-4 words
      const words: Array<{ text: string, start: number, end: number }> = []
      const regex = /\S+/g
      let match
      while ((match = regex.exec(text)) !== null) {
        words.push({ text: match[0], start: match.index, end: match.index + match[0].length })
      }

      // Generate 2-4 word phrases
      for (let windowSize = 2; windowSize <= 4; windowSize++) {
        for (let i = 0; i <= words.length - windowSize; i++) {
          const phraseWords = words.slice(i, i + windowSize)
          const phraseText = phraseWords.map(w => w.text).join(' ')
          const start = phraseWords[0].start
          const end = phraseWords[phraseWords.length - 1].end
          results.push({ text: phraseText, position: [start, end] })
        }
      }
    }

    return results
  }

  /**
   * Get comprehensive index statistics
   *
   * Returns detailed stats about all internal indexes including
   * entity counts, vector index size, graph relationships, and
   * estimated memory usage.
   *
   * @returns Index statistics object
   *
   * @example
   * const stats = await brain.indexStats()
   * console.log(`Entities: ${stats.entities}`)
   * console.log(`Vectors: ${stats.vectors}`)
   * console.log(`Relationships: ${stats.relationships}`)
   */
  async indexStats(): Promise<{
    entities: number
    vectors: number
    relationships: number
    metadataFields: string[]
    memoryUsage: {
      vectors: number
      graph: number
      metadata: number
      total: number
    }
  }> {
    await this.ensureInitialized()

    const metadataStats = await this.metadataIndex.getStats()
    const graphStats = this.graphIndex.getStats()
    const vectorCount = this.index.size()

    // Get unique metadata field names
    const metadataFields = metadataStats.fieldsIndexed || []

    return {
      entities: metadataStats.totalEntries,
      vectors: vectorCount,
      relationships: graphStats.totalRelationships,
      metadataFields,
      memoryUsage: {
        vectors: vectorCount * 384 * 4, // 384 dimensions * 4 bytes per float32
        graph: graphStats.memoryUsage,
        metadata: metadataStats.indexSize || 0,
        total: (vectorCount * 384 * 4) + graphStats.memoryUsage + (metadataStats.indexSize || 0)
      }
    }
  }

  /**
   * Validate metadata index consistency and detect corruption
   *
   * Returns health status and recommendations for repair. Corruption typically
   * manifests as high avg entries/entity (expected ~30, corrupted can be 100+)
   * caused by the update() field asymmetry bug (fixed).
   *
   * @returns Promise resolving to validation results
   *
   * @example
   * const validation = await brain.validateIndexConsistency()
   * if (!validation.healthy) {
   *   console.log(validation.recommendation)
   *   // Run brain.rebuildIndex() to repair
   * }
   */
  async validateIndexConsistency(): Promise<{
    healthy: boolean
    avgEntriesPerEntity: number
    entityCount: number
    indexEntryCount: number
    recommendation: string | null
    /** Cross-layer: each provider's own invariant self-report, when
     *  it exposes validateInvariants(). Absent providers are simply not listed. */
    providers?: ProviderInvariantReport[]
  }> {
    await this.ensureInitialized()
    const result = await this.metadataIndex.validateConsistency()

    // Cross-layer integrity: validateConsistency() above only sees the
    // JS metadata index — it is BLIND to a native provider whose manifest ↔
    // segments ↔ counts have diverged. Delegate to each provider's own
    // validateInvariants() and aggregate, so "healthy-while-broken" is impossible.
    const providers = await this.collectProviderInvariants()
    const brokenProviders = providers.filter((p) => !p.healthy)

    let healthy = result.healthy
    const notes: string[] = []
    if (result.recommendation) notes.push(result.recommendation)

    if (this._indexRebuildFailed) {
      healthy = false
      notes.unshift(
        `Index rebuild failed at init() (degraded — queries may be incomplete): ` +
          `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.`
      )
    }
    if (this._indexDegradedIds.size > 0) {
      healthy = false
      notes.unshift(
        `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
          `failed-rollback recovery — the derived index may be incomplete for them. ` +
          `Run repairIndex() to reconcile.`
      )
    }
    for (const p of brokenProviders) {
      healthy = false
      const failing = p.invariants.filter((i) => !i.holds)
      notes.unshift(
        `Provider '${p.provider}' reports ${failing.length} failing invariant(s): ` +
          failing.map((i) => `${i.name} (${i.detail}; heal=${i.heal})`).join('; ') +
          `. Run repairIndex() to reconcile from canonical.`
      )
    }

    return {
      ...result,
      healthy,
      recommendation: notes.length > 0 ? notes.join(' ') : null,
      ...(providers.length > 0 ? { providers } : {})
    }
  }

  /**
   * @description Feature-detect + call each index provider's OPTIONAL
   * `validateInvariants()` and collect the reports. The contract is
   * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a
   * provider violates that, the throw is turned into an UNHEALTHY synthetic
   * report (loud), never swallowed into "healthy". Providers that do not expose
   * the hook are simply omitted (the JS baseline validates via
   * `metadataIndex.validateConsistency()`).
   * @returns One report per provider that exposes `validateInvariants()`.
   */
  private async collectProviderInvariants(): Promise<ProviderInvariantReport[]> {
    const reports: ProviderInvariantReport[] = []
    const providers: unknown[] = [this.metadataIndex, this.index, this.graphIndex]
    for (const provider of providers) {
      const fn = (provider as { validateInvariants?: () => Promise<ProviderInvariantReport> } | null)
        ?.validateInvariants
      if (typeof fn !== 'function') continue
      try {
        const report = await fn.call(provider)
        if (report && Array.isArray(report.invariants)) reports.push(report)
      } catch (err) {
        // ONE CONTRACT FOR A THROWING PROBE, both engines: a probe that throws
        // is `heal: 'none'` with the error in `detail` — flakiness can never
        // buy a rebuild, and a thrown check never changes `serving` (the
        // provider's serving verdict is composed by the provider, not inferred
        // from a probe that failed to run). This catch used to synthesize
        // `heal: 'rebuild'` — the read-triggered dark-rebuild lever one
        // transient exception away — while the native composer said 'none';
        // two components disagreeing on what a throw means is how a flaky
        // probe became an outage. `healthy: false` stays: an unrunnable probe
        // is a named, loud, unverified state, never a clean bill.
        const name = typeof (provider as { name?: string })?.name === 'string'
          ? (provider as { name: string }).name
          : 'unknown'
        reports.push({
          provider: name,
          healthy: false,
          serving: true,
          invariants: [
            {
              name: 'validate-invariants-threw',
              holds: false,
              detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`,
              heal: 'none'
            }
          ],
          checkedAt: Date.now(),
          durationMs: 0
        })
      }
    }
    return reports
  }

  /**
   * Get metadata index statistics
   *
   * Returns detailed statistics about the metadata index including
   * total entries, IDs indexed, and fields indexed.
   *
   * @returns Promise resolving to index statistics
   */
  async getIndexStats(): Promise<{
    totalEntries: number
    totalIds: number
    fieldsIndexed: string[]
    lastRebuild: number
    indexSize: number
  }> {
    await this.ensureInitialized()
    return this.metadataIndex.getStats()
  }

  /**
   * Get graph neighbors of an entity
   *
   * Traverses the relationship graph to find connected entities.
   * Supports filtering by direction and relationship type.
   *
   * @param entityId The entity to get neighbors for
   * @param options Optional traversal options
   * @returns Array of neighbor entity IDs
   *
   * @example
   * // Get all connected entities
   * const allNeighbors = await brain.neighbors(entityId)
   *
   * // Get only outgoing connections
   * const outgoing = await brain.neighbors(entityId, { direction: 'outgoing' })
   *
   * // Get incoming connections with specific verb type
   * const incoming = await brain.neighbors(entityId, {
   *   direction: 'incoming',
   *   verbType: VerbType.RELATES_TO
   * })
   */
  async neighbors(
    entityId: string,
    options?: {
      direction?: 'outgoing' | 'incoming' | 'both'
      depth?: number
      verbType?: VerbType | VerbType[]
      limit?: number
    }
  ): Promise<string[]> {
    // Graph read (see related): adjacency traversal only.
    await this.ensureInitialized({ needs: ['graph'] })
    await this.verifyGraphAdjacencyLive()

    const direction = options?.direction || 'both'
    const limit = options?.limit
    const verbTypes = options?.verbType === undefined
      ? undefined
      : new Set(Array.isArray(options.verbType) ? options.verbType : [options.verbType])

    // Map our API direction to graphIndex direction
    const graphDirection = direction === 'outgoing' ? 'out' :
                           direction === 'incoming' ? 'in' : 'both'

    let neighbors = await this.getTypedNeighbors(entityId, graphDirection, verbTypes, limit)

    // Handle depth > 1 (multi-hop traversal). The verb-type filter is applied at EVERY hop
    // (previously only the first), and the BFS is bounded by `limit` so a dense graph can't
    // expand without limit before the final slice.
    if (options?.depth && options.depth > 1) {
      const visited = new Set<string>([entityId, ...neighbors])
      let currentLevel = neighbors

      for (let d = 1; d < options.depth; d++) {
        if (limit && neighbors.length >= limit) break
        const nextLevel: string[] = []

        outer: for (const nodeId of currentLevel) {
          const nodeNeighbors = await this.getTypedNeighbors(nodeId, graphDirection, verbTypes)
          for (const neighbor of nodeNeighbors) {
            if (!visited.has(neighbor)) {
              visited.add(neighbor)
              nextLevel.push(neighbor)
              neighbors.push(neighbor)
              if (limit && neighbors.length >= limit) break outer
            }
          }
        }

        if (nextLevel.length === 0) break
        currentLevel = nextLevel
      }

      if (limit && neighbors.length > limit) {
        neighbors = neighbors.slice(0, limit)
      }
    }

    return neighbors
  }

  /**
   * Neighbours of a single node, optionally filtered to a set of verb types.
   *
   * Extracted so depth traversal can apply the verb-type filter at every hop (not just the
   * first). For `direction: 'both'` the verb scan unions out-edges and in-edges so the filter
   * isn't silently limited to outgoing edges.
   */
  private async getTypedNeighbors(
    nodeId: string,
    graphDirection: 'in' | 'out' | 'both',
    verbTypes?: Set<VerbType>,
    limit?: number
  ): Promise<string[]> {
    // READ-SURFACE READINESS GATE (the 4.2.4 blackout's brainy half): every
    // index read funnels through this helper, so the gate here makes
    // serve-while-not-ready UNREPRESENTABLE — a production store once acked
    // writes while every non-find() read served empty from a not-ready
    // provider for 15 minutes. A CHECK only — it never builds; throws a typed
    // NotReady error if a provider's health report says it isn't serving.
    this.ensureIndexesLoaded(['graph'])
    // 8.0 BigInt boundary: unmapped node → no relations.
    const nodeInt = this.graphEntityInt(nodeId)
    if (nodeInt === undefined) return []

    const neighbors = this.entityIntsToUuids(
      await this.graphIndex.getNeighbors(nodeInt, { direction: graphDirection, limit })
    )
    if (!verbTypes || verbTypes.size === 0) return neighbors

    // Gather candidate edges in the relevant direction(s).
    const verbInts: bigint[] = []
    if (graphDirection !== 'in') verbInts.push(...await this.graphIndex.getVerbIdsBySource(nodeInt))
    if (graphDirection !== 'out') verbInts.push(...await this.graphIndex.getVerbIdsByTarget(nodeInt))
    const verbIds = await this.resolveVerbIntsToIds(verbInts)

    const verbs = await this.graphIndex.getVerbsBatchCached(verbIds)
    const neighborSet = new Set(neighbors)
    const filtered: string[] = []
    for (const [, verb] of verbs) {
      if (verbTypes.has(verb.type as VerbType) || verbTypes.has(verb.verb as VerbType)) {
        const neighborId = verb.sourceId === nodeId ? verb.targetId : verb.sourceId
        if (neighborSet.has(neighborId)) filtered.push(neighborId)
      }
    }
    return filtered
  }

  /**
   * Find semantic duplicates in the database
   *
   * Uses embedding similarity to identify entities that may be
   * duplicates or near-duplicates based on their content.
   *
   * @param options Optional search options
   * @returns Array of duplicate groups with similarity scores
   *
   * @example
   * // Find all duplicates with default threshold (0.85)
   * const duplicates = await brain.findDuplicates()
   *
   * // Find duplicates of a specific type with custom threshold
   * const personDupes = await brain.findDuplicates({
   *   type: NounType.PERSON,
   *   threshold: 0.9,
   *   limit: 100
   * })
   */
  async findDuplicates(options?: {
    threshold?: number
    type?: NounType
    limit?: number
  }): Promise<Array<{
    entity: Entity<any>
    duplicates: Array<{ entity: Entity<any>; similarity: number }>
  }>> {
    await this.ensureInitialized()

    const threshold = options?.threshold ?? 0.85
    const limit = options?.limit ?? 100

    // Get entities to check
    const findParams: FindParams<any> = {
      limit: Math.min(limit * 10, 1000), // Get more entities to find duplicates within
      type: options?.type
    }

    const entities = await this.find(findParams)
    const results: Array<{
      entity: Entity<any>
      duplicates: Array<{ entity: Entity<any>; similarity: number }>
    }> = []

    const processedIds = new Set<string>()

    for (const result of entities) {
      if (processedIds.has(result.id)) continue

      // Find similar entities
      const similar = await this.similar({
        to: result.id,
        limit: 20, // Check top 20 similar entities
        type: options?.type
      })

      // Filter to those above threshold (excluding self)
      const duplicates = similar
        .filter(s => s.id !== result.id && s.score >= threshold)
        .map(s => ({
          entity: s.entity,
          similarity: s.score
        }))

      if (duplicates.length > 0) {
        results.push({
          entity: result.entity,
          duplicates
        })

        // Mark all duplicates as processed to avoid reverse matches
        duplicates.forEach(d => processedIds.add(d.entity.id))
      }

      processedIds.add(result.id)

      // Stop if we have enough results
      if (results.length >= limit) break
    }

    return results
  }

  /**
   * Cluster entities by semantic similarity
   *
   * Groups entities into clusters based on their embedding similarity.
   * Uses a greedy algorithm that finds densely connected components
   * using the HNSW index for efficient neighbor lookup.
   *
   * @param options Optional clustering options
   * @returns Array of clusters with entities and optional centroids
   *
   * @example
   * // Find all clusters with default threshold
   * const clusters = await brain.cluster()
   *
   * // Find document clusters with higher threshold
   * const docClusters = await brain.cluster({
   *   type: NounType.Document,
   *   threshold: 0.85,
   *   minClusterSize: 3
   * })
   *
   * for (const cluster of docClusters) {
   *   console.log(`Cluster ${cluster.clusterId}: ${cluster.entities.length} entities`)
   * }
   */
  async cluster(options?: {
    threshold?: number
    type?: NounType
    minClusterSize?: number
    limit?: number
    includeCentroid?: boolean
  }): Promise<Array<{
    clusterId: string
    entities: Entity<any>[]
    centroid?: number[]
  }>> {
    await this.ensureInitialized()

    const threshold = options?.threshold ?? 0.8
    const minClusterSize = options?.minClusterSize ?? 2
    const limit = options?.limit ?? 100
    const includeCentroid = options?.includeCentroid ?? false

    // Get entities to cluster
    const findParams: FindParams<any> = {
      limit: 1000, // Process up to 1000 entities
      type: options?.type
    }

    const allEntities = await this.find(findParams)
    const clustered = new Set<string>()
    const clusters: Array<{
      clusterId: string
      entities: Entity<any>[]
      centroid?: number[]
    }> = []

    // Greedy clustering: for each unclustered entity, find its similar neighbors
    for (const result of allEntities) {
      if (clustered.has(result.id)) continue

      // Find similar entities to this one
      const similar = await this.similar({
        to: result.id,
        limit: 50,
        threshold,
        type: options?.type
      })

      // Filter to unclustered entities (including self)
      const clusterMembers = similar.filter(s => !clustered.has(s.id))

      // Only create cluster if it meets minimum size
      if (clusterMembers.length >= minClusterSize) {
        const entities = clusterMembers.map(s => s.entity)

        // Mark all as clustered
        clusterMembers.forEach(s => clustered.add(s.id))

        // Calculate centroid if requested
        let centroid: number[] | undefined
        if (includeCentroid && entities.length > 0) {
          const vectors = entities
            .filter(e => e.vector && e.vector.length > 0)
            .map(e => e.vector as number[])

          if (vectors.length > 0) {
            // Average all vectors to get centroid
            const dim = vectors[0].length
            centroid = new Array(dim).fill(0)
            for (const vec of vectors) {
              for (let i = 0; i < dim; i++) {
                centroid[i] += vec[i]
              }
            }
            for (let i = 0; i < dim; i++) {
              centroid[i] /= vectors.length
            }
          }
        }

        clusters.push({
          clusterId: `cluster-${clusters.length + 1}`,
          entities,
          centroid
        })

        if (clusters.length >= limit) break
      } else {
        // Mark single entity as processed (not in a cluster)
        clustered.add(result.id)
      }
    }

    return clusters
  }

  /**
   * Storage adapter (internal API)
   * Direct access to the underlying BaseStorage for diagnostics and tests
   * (e.g. reading the transaction log). Not part of the supported public
   * surface.
   * @internal
   */
  get storageAdapter(): BaseStorage {
    return this.storage
  }

  // ============= HELPER METHODS =============

  /**
   * Parse natural language query using advanced NLP with 220+ patterns
   * The embedding model is always available as it's core to Brainy's functionality
   */
  private async parseNaturalQuery(query: string): Promise<FindParams<T>> {
    // Initialize NLP processor if needed (lazy loading)
    if (!this._nlp) {
      this._nlp = new NaturalLanguageProcessor(this)
      await this._nlp.init() // Ensure pattern library is loaded
    }
    
    // Process with our advanced pattern library (220+ patterns with embeddings)
    const tripleQuery = await this._nlp.processNaturalQuery(query)
    
    // Convert TripleQuery to FindParams
    const params: FindParams<T> = {}
    
    // Handle vector search
    if (tripleQuery.like || tripleQuery.similar) {
      params.query = typeof tripleQuery.like === 'string' ? tripleQuery.like : 
                     typeof tripleQuery.similar === 'string' ? tripleQuery.similar : query
    } else if (!tripleQuery.where && !tripleQuery.connected) {
      // Default to vector search if no other criteria specified
      params.query = query
    }
    
    // Handle metadata filtering
    if (tripleQuery.where) {
      params.where = tripleQuery.where as Partial<T>
    }
    
    // Handle graph relationships
    if (tripleQuery.connected) {
      params.connected = {
        to: Array.isArray(tripleQuery.connected.to) ? tripleQuery.connected.to[0] : tripleQuery.connected.to,
        from: Array.isArray(tripleQuery.connected.from) ? tripleQuery.connected.from[0] : tripleQuery.connected.from,
        via: tripleQuery.connected.type as VerbType | undefined,
        depth: tripleQuery.connected.depth,
        direction: tripleQuery.connected.direction
      }
    }
    
    // Handle other options
    if (tripleQuery.limit) params.limit = tripleQuery.limit
    if (tripleQuery.offset) params.offset = tripleQuery.offset
    
    return this.enhanceNLPResult(params, query)
  }
  
  /**
   * Enhance NLP results with fusion scoring
   */
  private enhanceNLPResult(params: FindParams<T>, _originalQuery: string): FindParams<T> {
    // Add fusion scoring for complex queries
    if (params.query && params.where && Object.keys(params.where).length > 0) {
      params.fusion = params.fusion || {
        strategy: 'adaptive',
        weights: {
          vector: 0.6,
          field: 0.3,
          graph: 0.1
        }
      }
    }
    return params
  }

  /**
   * @description Build the MetadataIndex filter object from a query's structured
   * criteria (`where` / `type` / `subtype` / `service` / `excludeVFS`) — the shared
   * filter shape consumed by `getIdsForFilter` / `getIdSetForFilter`. Applies the
   * `where.type → noun` alias and expands a multi-type filter into an `anyOf`.
   * @param params - The structured query criteria.
   * @returns The filter object, or `null` when no structured criteria are present.
   */
  private buildMetadataFilter(params: {
    where?: any
    type?: NounType | NounType[]
    subtype?: string | string[]
    service?: string
    excludeVFS?: boolean
  }): any | null {
    // An empty `where: {}` carries no predicates — it is NOT structured
    // criteria (see whereConstrains). Counting it would produce an empty
    // filter object, and getIdsForFilter({}) / getIdSetForFilter({}) answer
    // the empty set by contract — silently emptying a match-all query.
    const constrainingWhere = whereConstrains(params.where) ? params.where : undefined
    if (!(constrainingWhere || params.type || params.subtype || params.service || params.excludeVFS)) {
      return null
    }
    let filter: any = {}
    if (constrainingWhere) {
      // Where keys pass through UNTOUCHED — the one addressing law parses
      // them at the index boundary (bare = user metadata, system.* = engine
      // scalars, typed refusal otherwise). The old type→noun alias is dead.
      Object.assign(filter, constrainingWhere)
    }
    if (params.service) filter['system.service'] = params.service
    if (params.excludeVFS === true) {
      filter.vfsType = { exists: false }
      filter.isVFSEntity = { ne: true }
    }
    // Subtype (top-level standard field — fast path). Assigned BEFORE the type-array
    // expansion below so the spread into each anyOf branch carries it through.
    if (params.subtype !== undefined) {
      filter['system.subtype'] = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype
    }
    if (params.type) {
      const types = Array.isArray(params.type) ? params.type : [params.type]
      if (types.length === 1) {
        filter['system.type'] = types[0]
      } else {
        filter = { anyOf: types.map((type) => ({ 'system.type': type, ...filter })) }
      }
    }
    return filter
  }

  /**
   * Execute vector search component
   *
   * @param params Find parameters
   * @param candidateIds Optional pre-resolved metadata filter IDs for metadata-first search.
   *   When provided, HNSW search is restricted to these candidates:
   *   - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering)
   *   - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups
   * @param allowedIds Optional 8.0 #46 predicate-pushdown universe as an
   *   {@link OpaqueIdSet} (native/cor roaring Buffer) — forwarded straight into the
   *   beam walk with no id materialization. The native vector provider consumes it;
   *   the JS index ignores the opaque form and relies on `candidateIds`.
   */
  private async executeVectorSearch(
    params: FindParams<T>,
    candidateIds?: string[],
    allowedIds?: OpaqueIdSet
  ): Promise<Result<T>[]> {
    // Vector cold-read guard: before trusting a semantic/vector result, verify the
    // vector index actually SERVES a known persisted vector (one-shot per brain).
    // A pure semantic find({ query }) has no filter, so verifyMetadataLive never
    // fires — a cold native index that loaded its COUNT but not its serving
    // structure would return a silent []. This self-heals (rebuild) or throws
    // VectorIndexNotReadyError instead.
    await this.verifyVectorLive()

    const vector = params.vector || (await this.embed(params.query!))
    const limit = params.limit || 10

    // Build search options for metadata-first candidate filtering. Pass both forms
    // when available: the native provider prefers the opaque `allowedIds` (zero
    // crossing), the JS index uses the materialized `candidateIds`.
    const searchOptions =
      candidateIds || allowedIds
        ? {
            ...(candidateIds && { candidateIds }),
            ...(allowedIds && { allowedIds })
          }
        : undefined

    // HNSW search with optional metadata-first candidate filtering
    const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)

    // Batch-load entities for 10-50x faster cloud storage performance
    // GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
    const ids = searchResults.map(([id]) => id)
    const entitiesMap = await this.batchGet(ids)

    const results: Result<T>[] = []
    for (const [id, distance] of searchResults) {
      const entity = entitiesMap.get(id)
      if (entity) {
        const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
        results.push(this.createResult(id, score, entity))
      }
    }

    return results
  }
  
  /**
   * Execute proximity search component
   */
  private async executeProximitySearch(params: FindParams<T>): Promise<Result<T>[]> {
    if (!params.near) return []

    // Vector cold-read guard (see executeVectorSearch): proximity search also
    // hits this.index.search — verify it serves before trusting an empty result.
    await this.verifyVectorLive()

    // Teaching error: without an anchor id the constraint is meaningless, and
    // letting it fall through produces an opaque storage-layer sharding error.
    if (!params.near.id) {
      throw new Error(
        "find({ near }): 'near.id' is required — pass the entity to search around, " +
          'e.g. near: { id, threshold }. To impose a minimum score on plain semantic ' +
          'results, filter on result.score instead.'
      )
    }

    const nearEntity = await this.get(params.near.id)
    if (!nearEntity) return []

    const nearResults: [string, number][] = await this.index.search(nearEntity.vector, params.limit || 10)

    // Filter by threshold first to minimize batch fetch
    const threshold = params.near.threshold || 0.7
    const filteredResults = nearResults.filter(([, distance]) => {
      const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
      return score >= threshold
    })

    // Batch-load entities for 10-50x faster cloud storage performance
    const ids = filteredResults.map(([id]) => id)
    const entitiesMap = await this.batchGet(ids)

    const results: Result<T>[] = []
    for (const [id, distance] of filteredResults) {
      const entity = entitiesMap.get(id)
      if (entity) {
        const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
        results.push(this.createResult(id, score, entity))
      }
    }

    return results
  }
  
  /**
   * Execute graph search component.
   *
   * Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via
   * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read
   * only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via`
   * were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the
   * immediate neighbour at every depth.
   */
  private async executeGraphSearch(params: FindParams<T>, existingResults: Result<T>[]): Promise<Result<T>[]> {
    if (!params.connected) return existingResults

    const { from, to, depth, direction = 'both' } = params.connected
    const via = params.connected.via ?? params.connected.type
    const subtypeFilter = params.connected.subtype
    const effectiveDepth = depth ?? 1

    // GraphConstraints speaks 'in' | 'out' | 'both'; neighbors() speaks 'incoming' | 'outgoing' | 'both'.
    const toNeighborDir = (d: 'in' | 'out' | 'both'): 'incoming' | 'outgoing' | 'both' =>
      d === 'in' ? 'incoming' : d === 'out' ? 'outgoing' : 'both'

    const connectedIds = new Set<string>()

    // Population is extracted into a closure so it can be re-run VERBATIM after a
    // cold-load heal (verdict 'rebuilt') — re-executing the SAME traversal once the
    // adjacency is actually loaded, without changing any collection semantics.
    const populate = async (): Promise<void> => {
      connectedIds.clear()
      if (subtypeFilter !== undefined) {
        // Multi-hop BFS with per-hop (verbType, subtype) predicate. Works on the
        // open-core JS path at any depth. When a graph-index provider exposes a
        // faster `findConnectedSubtype` (native BFS with the same semantics),
        // `nativeSubtypeBfs` routes through it transparently — same pattern as
        // every other provider hook.
        const subtypeArr = Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter]
        const subtypeSet = new Set(subtypeArr)

        // Native fast path (D.3): single verb type + single subtype + outgoing
        // walk match the native BFS semantics exactly (out-edges only, source
        // excluded, visited-set cycle guard). Entity ints in, entity ints out —
        // UUID conversion stays at this boundary. Returns null when the query
        // shape (or provider) can't take the native route.
        const nativeSubtypeBfs = async (
          anchor: string,
          walk: 'in' | 'out' | 'both'
        ): Promise<Set<string> | null> => {
          if (walk !== 'out') return null
          if (via === undefined || Array.isArray(via)) return null
          if (subtypeArr.length !== 1) return null
          const provider = this.graphIndex as Partial<{
            findConnectedSubtype(
              sourceInt: bigint,
              verbTypeIndex: number,
              subtype: string | null,
              depth: number,
              limit?: number | null
            ): Promise<bigint[]>
          }>
          if (typeof provider.findConnectedSubtype !== 'function') return null

          const anchorInt = this.graphEntityInt(anchor)
          if (anchorInt === undefined) return new Set() // unmapped → no relations

          const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
          // No limit: match the JS BFS exactly — overall result limiting happens
          // downstream against existingResults.
          const reachedInts = await provider.findConnectedSubtype(
            anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
          )
          return new Set(this.entityIntsToUuids(reachedInts))
        }

        const bfsWithSubtype = async (anchor: string, walk: 'in' | 'out' | 'both'): Promise<Set<string>> => {
          const nativeResult = await nativeSubtypeBfs(anchor, walk)
          if (nativeResult !== null) return nativeResult

          const visited = new Set<string>([anchor])
          const reached = new Set<string>()
          let frontier = new Set<string>([anchor])

          for (let hop = 0; hop < effectiveDepth && frontier.size > 0; hop++) {
            const nextFrontier = new Set<string>()
            for (const node of frontier) {
              // Walk outgoing edges (when direction includes outbound) and incoming
              // edges (when direction includes inbound). Both = union.
              const dirs: Array<'from' | 'to'> = []
              if (walk === 'out' || walk === 'both') dirs.push('from')
              if (walk === 'in' || walk === 'both') dirs.push('to')

              for (const dir of dirs) {
                const edges = await this.related({
                  ...(dir === 'from' ? { from: node } : { to: node }),
                  ...(via && { type: via as VerbType | VerbType[] }),
                  subtype: subtypeArr,
                  limit: 10000
                })
                for (const edge of edges) {
                  // Defensive: related() honors the subtype filter, but check
                  // again in case a future impl widens it.
                  if (edge.subtype !== undefined && !subtypeSet.has(edge.subtype)) continue
                  const neighbor = dir === 'from' ? edge.to : edge.from
                  if (visited.has(neighbor)) continue
                  visited.add(neighbor)
                  reached.add(neighbor)
                  nextFrontier.add(neighbor)
                }
              }
            }
            frontier = nextFrontier
          }
          return reached
        }

        if (from) {
          for (const id of await bfsWithSubtype(from, direction)) connectedIds.add(id)
        }
        if (to) {
          const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both'
          for (const id of await bfsWithSubtype(to, reverse)) connectedIds.add(id)
        }
      } else {
        // No subtype filter — fast path via neighbors(), which does the same
        // depth-aware BFS but only filters by verbType.
        const collect = (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise<string[]> =>
          this.neighbors(id, { direction: dir, depth: effectiveDepth, verbType: via })

        if (from) {
          for (const id of await collect(from, toNeighborDir(direction))) connectedIds.add(id)
        }

        if (to) {
          const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both'
          for (const id of await collect(to, toNeighborDir(reverse))) connectedIds.add(id)
        }
      }
    }

    await populate()

    // Cold-load guard: an empty connected set is suspicious. The native adjacency can report
    // size()>0 (or isReady()===false) on a cold open yet have loaded NO source→target edges — so
    // traversal would silently return [] as if it were truth. Re-verify against the health-report/
    // isReady() authority (or, for older providers, a READ-ONLY GLOBAL known-edge sample — NOT the
    // queried anchor, which may be genuinely edgeless): a dead adjacency throws
    // GraphIndexNotReadyError here rather than serving the empty set as fact — verifyGraphAdjacencyLive
    // never rebuilds, so a genuinely edgeless anchor simply verifies 'live' and the empty result stands.
    if (connectedIds.size === 0) {
      await this.verifyGraphAdjacencyLive()
    }

    // Filter existing results to only connected entities
    if (existingResults.length > 0) {
      return existingResults.filter(r => connectedIds.has(r.id))
    }

    // Batch-load connected entities for fast cloud-storage performance
    const results: Result<T>[] = []
    const ids = [...connectedIds]
    const entitiesMap = await this.batchGet(ids)
    for (const id of ids) {
      const entity = entitiesMap.get(id)
      if (entity) {
        results.push(this.createResult(id, 1.0, entity))
      }
    }

    return results
  }
  
  /**
   * Apply fusion scoring for multi-source results
   */
  private applyFusionScoring(results: Result<T>[], fusionType: any): Result<T>[] {
    // Implement different fusion strategies
    const strategy = typeof fusionType === 'string' ? fusionType : fusionType.strategy || 'weighted'
    
    switch (strategy) {
      case 'max':
        // Use maximum score from any source
        return results
        
      case 'average':
        // Average scores from multiple sources
        const scoreMap = new Map<string, number[]>()
        for (const result of results) {
          const scores = scoreMap.get(result.id) || []
          scores.push(result.score)
          scoreMap.set(result.id, scores)
        }
        
        return results.map(r => ({
          ...r,
          score: scoreMap.get(r.id)!.reduce((a, b) => a + b, 0) / scoreMap.get(r.id)!.length
        }))
        
      case 'weighted':
      default:
        // Weighted combination based on source importance
        const weights = fusionType.weights || { vector: 0.7, metadata: 0.2, graph: 0.1 }
        return results.map(r => ({
          ...r,
          score: r.score * (weights.vector || 1.0)
        }))
    }
  }

  /**
   * Execute text search using word index
   *
   * Performs keyword-based search using the __words__ index in MetadataIndexManager.
   * Returns results ranked by word match count.
   *
   * @param query - Text query to search for
   * @param limit - Maximum results to return
   * @returns Array of Results with scores based on match count
   */
  private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
    const textMatches = await this.metadataIndex.getIdsForTextQuery(query)
    if (textMatches.length === 0) return []

    // Take top matches and load entities
    const topMatches = textMatches.slice(0, limit * 2)  // Get more for filtering
    const ids = topMatches.map(m => m.id)
    const entitiesMap = await this.batchGet(ids)

    // Create results with scores based on match count
    const maxMatches = topMatches[0]?.matchCount || 1
    const results: Result<T>[] = []

    for (const match of topMatches) {
      const entity = entitiesMap.get(match.id)
      if (entity) {
        // Normalize score to 0-1 range based on match count
        const score = match.matchCount / maxMatches
        results.push(this.createResult(match.id, score, entity))
      }
    }

    return results
  }

  /**
   * Auto-detect optimal alpha for hybrid search
   *
   * Short queries (1-2 words) favor text search (lower alpha)
   * Long queries (5+ words) favor semantic search (higher alpha)
   *
   * @param query - The search query
   * @returns Alpha value between 0 (text only) and 1 (semantic only)
   */
  private autoAlpha(query: string): number {
    const wordCount = query.trim().split(/\s+/).filter(w => w.length > 0).length
    if (wordCount <= 2) return 0.3      // Favor text for short queries
    if (wordCount <= 5) return 0.5      // Balanced
    return 0.7                          // Favor semantic for long queries
  }

  /**
   * Reciprocal Rank Fusion (RRF) for combining search results
   *
   * RRF is a proven fusion algorithm that:
   * - Doesn't require score normalization
   * - Handles different score distributions
   * - Gives higher weight to top-ranked items
   *
   * Formula: score(d) = sum(1 / (k + rank(d))) for each list
   *
   * Now includes match visibility (textMatches, textScore, semanticScore, matchSource)
   *
   * @param textResults - Results from text search
   * @param semanticResults - Results from semantic search
   * @param alpha - Weight for semantic (0=text only, 1=semantic only)
   * @param queryWords - Original query words for match tracking
   * @param k - RRF constant (default: 60, standard in literature)
   * @returns Fused results sorted by combined score with match visibility
   */
  private async rrfFusion(
    textResults: Result<T>[],
    semanticResults: Result<T>[],
    alpha: number,
    queryWords: string[],
    k: number = 60
  ): Promise<Result<T>[]> {
    // Track scores and match details per entity
    interface MatchData {
      rrf: number
      textScore?: number
      semanticScore?: number
      textMatches: string[]
      hasText: boolean
      hasSemantic: boolean
    }
    const matchData = new Map<string, MatchData>()
    const entityMap = new Map<string, Entity<T>>()

    // Text contribution (1 - alpha weight)
    const textWeight = 1 - alpha
    textResults.forEach((r, rank) => {
      const rrfScore = textWeight * (1 / (k + rank + 1))
      const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
      existing.rrf += rrfScore
      existing.textScore = r.score  // Original text search score (0-1)
      existing.hasText = true
      matchData.set(r.id, existing)
      if (r.entity) entityMap.set(r.id, r.entity)
    })

    // Semantic contribution (alpha weight)
    semanticResults.forEach((r, rank) => {
      const rrfScore = alpha * (1 / (k + rank + 1))
      const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
      existing.rrf += rrfScore
      existing.semanticScore = r.score  // Original semantic search score (0-1)
      existing.hasSemantic = true
      matchData.set(r.id, existing)
      if (r.entity) entityMap.set(r.id, r.entity)
    })

    // Sort by fused score
    const sortedIds = Array.from(matchData.entries())
      .sort((a, b) => b[1].rrf - a[1].rrf)
      .map(([id, data]) => ({ id, data }))

    // Build results - need to load any missing entities
    const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id)
    if (missingIds.length > 0) {
      const loaded = await this.batchGet(missingIds)
      for (const [id, entity] of loaded) {
        entityMap.set(id, entity)
      }
    }

    // Performance: Build set of text result IDs for O(1) lookup
    // This avoids re-extracting text for entities that weren't in text results
    const textResultIds = new Set(textResults.map(r => r.id))

    // Create final results with match visibility
    const results: Result<T>[] = []
    for (const { id, data } of sortedIds) {
      const entity = entityMap.get(id)
      if (entity) {
        // Find which query words matched - uses fast path if entity wasn't in text results
        const textMatches = this.findMatchingWords(entity, queryWords, textResultIds)

        // Determine match source
        let matchSource: 'text' | 'semantic' | 'both'
        if (data.hasText && data.hasSemantic) {
          matchSource = 'both'
        } else if (data.hasText) {
          matchSource = 'text'
        } else {
          matchSource = 'semantic'
        }

        // Create result with match visibility
        const result = this.createResult(id, data.rrf, entity)
        result.textMatches = textMatches
        result.textScore = data.textScore
        result.semanticScore = data.semanticScore
        result.matchSource = matchSource

        results.push(result)
      }
    }

    return results
  }

  /**
   * Find which query words match in an entity's text content
   *
   * Performance: O(query_words × text_length) - only called when needed
   * At scale: Use textResultIds set for O(1) lookup instead of re-extracting
   *
   * @param entity - Entity to check
   * @param queryWords - Words from the search query
   * @param textResultIds - Optional: Set of IDs from text search (O(1) lookup)
   * @returns Array of matching query words
   */
  private findMatchingWords(
    entity: Entity<T>,
    queryWords: string[],
    textResultIds?: Set<string>
  ): string[] {
    // Fast path: if entity wasn't in text results, no words matched
    if (textResultIds && !textResultIds.has(entity.id)) {
      return []
    }

    // Slow path: extract text and check each word
    // Only happens for entities that DID match text search
    const textContent = this.metadataIndex.extractTextContent({
      data: entity.data,
      metadata: entity.metadata
    }).toLowerCase()

    return queryWords.filter(word => textContent.includes(word.toLowerCase()))
  }

  /**
   * Apply graph constraints using O(1) GraphAdjacencyIndex - TRUE Triple Intelligence!
   */
  private async applyGraphConstraints(
    results: Result<T>[],
    constraints: any
  ): Promise<Result<T>[]> {
    // Filter by graph connections using fast graph index
    if (constraints.to || constraints.from) {
      const filtered: Result<T>[] = []
      
      for (const result of results) {
        let hasConnection = false
        
        if (constraints.to) {
          // Check if this entity connects TO the target (O(1) lookup)
          const outgoingNeighbors = await this.getNeighborUuids(result.id, { direction: 'out' })
          hasConnection = outgoingNeighbors.includes(constraints.to)
        }

        if (constraints.from && !hasConnection) {
          // Check if this entity connects FROM the source (O(1) lookup)
          const incomingNeighbors = await this.getNeighborUuids(result.id, { direction: 'in' })
          hasConnection = incomingNeighbors.includes(constraints.from)
        }
        
        if (hasConnection) {
          filtered.push(result)
        }
      }
      
      return filtered
    }
    
    return results
  }

  /**
   * Convert storage-shape verbs to public `Relation`s. The storage layer has
   * already done the canonical reserved/custom split (every combine goes
   * through `hydrateVerbWithMetadata` — see src/types/reservedFields.ts), so
   * this is a pure top-level field mapping: `metadata` carries ONLY custom
   * fields, and every reserved field (`subtype`, `weight`, `confidence`,
   * timestamps, `service`, `data`) surfaces at top level.
   */
  private verbsToRelations(verbs: GraphVerb[]): Relation<T>[] {
    return verbs.map((v) => {
      return {
        id: v.id,
        from: v.sourceId,
        to: v.targetId,
        type: (v.verb || v.type) as VerbType,
        ...(v.subtype !== undefined && { subtype: v.subtype }),
        ...(v.visibility !== undefined && { visibility: v.visibility }),
        weight: v.weight ?? 1.0,
        ...(typeof v.confidence === 'number' && { confidence: v.confidence }),
        data: v.data,
        metadata: v.metadata,
        service: v.service as string,
        createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now(),
        ...(typeof v.updatedAt === 'number' && { updatedAt: v.updatedAt })
      }
    })
  }

  /**
   * Embed data into vector representation
   * Handles any data type by intelligently converting to string representation
   *
   * @param data - Any data to convert to vector (string, object, array, etc.)
   * @returns Promise that resolves to a numerical vector representation
   *
   * @example
   * // Basic string embedding
   * const vector = await brainy.embed('machine learning algorithms')
   * console.log('Vector dimensions:', vector.length)
   *
   * @example
   * // Object embedding with intelligent field extraction
   * const documentVector = await brainy.embed({
   *   title: 'AI Research Paper',
   *   content: 'This paper discusses neural networks...',
   *   author: 'Dr. Smith',
   *   category: 'machine-learning'
   * })
   * // Uses 'content' field for embedding by default
   *
   * @example
   * // Different object field priorities
   * // Priority: data > content > text > name > title > description
   * const vectors = await Promise.all([
   *   brainy.embed({ data: 'primary content' }),        // Uses 'data'
   *   brainy.embed({ content: 'main content' }),        // Uses 'content'
   *   brainy.embed({ text: 'text content' }),           // Uses 'text'
   *   brainy.embed({ name: 'entity name' }),            // Uses 'name'
   *   brainy.embed({ title: 'document title' }),        // Uses 'title'
   *   brainy.embed({ description: 'description text' }) // Uses 'description'
   * ])
   *
   * @example
   * // Array embedding for batch processing
   * const batchVectors = await brainy.embed([
   *   'first document',
   *   'second document',
   *   { content: 'third document as object' },
   *   { title: 'fourth document' }
   * ])
   * // Returns vector representing all items combined
   *
   * @example
   * // Complex object handling
   * const complexData = {
   *   user: { name: 'John', role: 'developer' },
   *   project: { name: 'AI Assistant', status: 'active' },
   *   metrics: { score: 0.95, performance: 'excellent' }
   * }
   * const vector = await brainy.embed(complexData)
   * // Converts entire object to JSON for embedding
   *
   * @example
   * // Pre-computing vectors for performance optimization
   * const documents = [
   *   { id: 'doc1', content: 'Document 1 content...' },
   *   { id: 'doc2', content: 'Document 2 content...' },
   *   { id: 'doc3', content: 'Document 3 content...' }
   * ]
   *
   * // Pre-compute all vectors
   * const vectors = await Promise.all(
   *   documents.map(doc => brainy.embed(doc.content))
   * )
   *
   * // Add entities with pre-computed vectors (faster)
   * for (let i = 0; i < documents.length; i++) {
   *   await brainy.add({
   *     data: documents[i],
   *     type: NounType.Document,
   *     vector: vectors[i] // Skip embedding computation
   *   })
   * }
   *
   * @example
   * // Custom embedding for search queries
   * async function searchWithCustomEmbedding(query: string) {
   *   // Enhance query for better matching
   *   const enhancedQuery = `search: ${query} relevant information`
   *   const queryVector = await brainy.embed(enhancedQuery)
   *
   *   // Use pre-computed vector for search
   *   return brainy.find({
   *     vector: queryVector,
   *     limit: 10
   *   })
   * }
   *
   * @example
   * // Handling edge cases gracefully
   * const edgeCases = await Promise.all([
   *   brainy.embed(null),           // Returns vector for empty string
   *   brainy.embed(undefined),      // Returns vector for empty string
   *   brainy.embed(''),             // Returns vector for empty string
   *   brainy.embed(42),             // Converts number to string
   *   brainy.embed(true),           // Converts boolean to string
   *   brainy.embed([]),             // Empty array handling
   *   brainy.embed({})              // Empty object handling
   * ])
   *
   * @example
   * // Using with similarity comparisons
   * const doc1Vector = await brainy.embed('artificial intelligence research')
   * const doc2Vector = await brainy.embed('machine learning algorithms')
   *
   * // Find entities similar to doc1Vector
   * const similar = await brainy.find({
   *   vector: doc1Vector,
   *   limit: 5
   * })
   */
  async embed(data: any): Promise<Vector> {
    // Handle different data types intelligently
    let textToEmbed: string | string[]

    if (typeof data === 'string') {
      textToEmbed = data
    } else if (Array.isArray(data)) {
      // Array of items - convert each to string
      textToEmbed = data.map(item => {
        if (typeof item === 'string') return item
        if (typeof item === 'number' || typeof item === 'boolean') return String(item)
        if (item && typeof item === 'object') {
          // For objects, try to extract meaningful text
          if (item.data) return String(item.data)
          if (item.content) return String(item.content)
          if (item.text) return String(item.text)
          if (item.name) return String(item.name)
          if (item.title) return String(item.title)
          if (item.description) return String(item.description)
          // Fallback to JSON for complex objects
          try {
            return JSON.stringify(item)
          } catch {
            return String(item)
          }
        }
        return String(item)
      })
    } else if (data && typeof data === 'object') {
      // Single object - extract meaningful text
      if (data.data) textToEmbed = String(data.data)
      else if (data.content) textToEmbed = String(data.content)
      else if (data.text) textToEmbed = String(data.text)
      else if (data.name) textToEmbed = String(data.name)
      else if (data.title) textToEmbed = String(data.title)
      else if (data.description) textToEmbed = String(data.description)
      else {
        // For complex objects, create a descriptive string
        try {
          textToEmbed = JSON.stringify(data)
        } catch {
          textToEmbed = String(data)
        }
      }
    } else if (data === null || data === undefined) {
      // Handle null/undefined gracefully
      textToEmbed = ''
    } else {
      // Numbers, booleans, etc - convert to string
      textToEmbed = String(data)
    }

    return this.embedder(textToEmbed)
  }

  /**
   * Eagerly load/fault-in the vector index, metadata index, and graph
   * adjacency so the FIRST real operation after this call runs at
   * steady-state cost — no page-cache-miss / demand-load latency on the
   * critical path. Complements {@link warmupEmbeddings} (which warms the
   * embedding engine, not the storage indexes).
   *
   * Sequence per surface:
   * - **Vector**: calls the provider's own `warm?()` when the active
   *   `'vector'` provider implements it (`'warmed'`). Otherwise runs a
   *   best-effort probe — one `search()` with a deterministic unit vector of
   *   the index's known dimension, `k = min(100, size())` — which faults in
   *   *some* backing storage as a side effect but is reported honestly as
   *   `'probed'`, never `'warmed'`. An empty index or unknown dimension has
   *   nothing to probe (`'unavailable'`).
   * - **Metadata**: calls the provider's own `warm?()` when the active
   *   `'metadataIndex'` provider implements it (`'warmed'`) — the seam a
   *   native metadata provider lights up so it is not duck-typed against the
   *   JS manager's method. Otherwise falls back to full hydration on the
   *   built-in JS manager — every persisted field's sparse index is loaded
   *   from storage (`MetadataIndexManager.hydrateAll()`), not just the
   *   heuristic common-fields subset `init()` warms — and reports `'warmed'`.
   *   Neither seam present → `'unavailable'` (honest: `init()` is never used
   *   as a substitute here, since a native provider's `init()` may be a
   *   cheap verify rather than a real warm).
   * - **Graph**: calls the provider's own `warm?()` when the active graph
   *   provider implements it; otherwise re-runs its existing eager cold-load
   *   `init()` seam (idempotent — the JS adjacency index's `init()` already
   *   loads the full LSM manifests + SSTables, so re-invoking it here is a
   *   genuine full hydration, not a stat call).
   *
   * A single surface's `'unavailable'`/probe outcome never fails the whole
   * call — `warm()` is a best-effort readiness step, not a correctness gate;
   * queries still demand-load normally regardless of what `warm()` achieved.
   *
   * @returns A {@link WarmReport}: honest per-surface outcome + timing.
   * @example
   * ```typescript
   * const brain = new Brainy({ storage: { path: '/data' } })
   * await brain.init()
   * const report = await brain.warm() // blocking, explicit control over timing
   * console.log(report.vector.outcome, report.vector.durationMs)
   *
   * // Or opt into the same work automatically during init():
   * const eager = new Brainy({ storage: { path: '/data' }, warmOnOpen: true })
   * await eager.init() // warm() already ran before this resolves
   * ```
   */
  async warm(): Promise<WarmReport> {
    await this.ensureInitialized({ needs: ['vector', 'metadata', 'graph'] })

    const totalStart = Date.now()

    // --- Vector ---------------------------------------------------------
    const vectorStart = Date.now()
    let vectorOutcome: WarmOutcome
    const vectorProvider = this.index as VectorIndexProvider & { warm?: () => Promise<void> }
    if (typeof vectorProvider.warm === 'function') {
      await vectorProvider.warm()
      vectorOutcome = 'warmed'
    } else {
      const size = this.index.size()
      const dimension = this.dimensions
      if (size > 0 && dimension && dimension > 0) {
        // A deterministic unit vector (L2 norm 1) — same probe every call,
        // no reliance on stored data shape.
        const probeVector = new Array(dimension).fill(1 / Math.sqrt(dimension))
        await this.index.search(probeVector, Math.min(100, size))
        vectorOutcome = 'probed'
      } else {
        // Nothing to probe: an empty index, or the vector dimension is not
        // yet known (no entity has ever been added).
        vectorOutcome = 'unavailable'
      }
    }
    const vectorDurationMs = Date.now() - vectorStart

    // --- Metadata --------------------------------------------------------
    const metadataStart = Date.now()
    let metadataOutcome: WarmOutcome
    const metadataProvider = this.metadataIndex as unknown as MetadataIndexProvider
    const metadataWithHydrate = this.metadataIndex as unknown as { hydrateAll?: () => Promise<void> }
    if (typeof metadataProvider.warm === 'function') {
      // Active provider (e.g. a native metadata index) declares its own warm
      // seam — route through it FIRST so a native provider's warmth is
      // reported honestly instead of being duck-typed against the JS
      // manager's hydrateAll(), which a native provider does not implement.
      await metadataProvider.warm()
      metadataOutcome = 'warmed'
    } else if (typeof metadataWithHydrate.hydrateAll === 'function') {
      // Built-in JS manager path — full sparse-index hydration.
      await metadataWithHydrate.hydrateAll()
      metadataOutcome = 'warmed'
    } else {
      // No hydration seam on this metadata provider — nothing to run. (No
      // init() fallback here: init() on a native provider may be a cheap
      // verify, and reporting that as warmth would lie.)
      metadataOutcome = 'unavailable'
    }
    const metadataDurationMs = Date.now() - metadataStart

    // --- Graph -------------------------------------------------------------
    const graphStart = Date.now()
    let graphOutcome: WarmOutcome
    const graphProvider = this.graphIndex as GraphIndexProvider & {
      warm?: () => Promise<void>
      init?: () => Promise<void>
    }
    if (typeof graphProvider.warm === 'function') {
      await graphProvider.warm()
      graphOutcome = 'warmed'
    } else if (typeof graphProvider.init === 'function') {
      // Existing eager cold-load seam (readiness contract): idempotent, and
      // for the JS adjacency index it's already a genuine full load of the
      // LSM manifests + SSTables — a real hydration, not a stat call.
      await graphProvider.init()
      graphOutcome = 'warmed'
    } else {
      graphOutcome = 'unavailable'
    }
    const graphDurationMs = Date.now() - graphStart

    return {
      vector: { outcome: vectorOutcome, durationMs: vectorDurationMs },
      metadata: { outcome: metadataOutcome, durationMs: metadataDurationMs },
      graph: { outcome: graphOutcome, durationMs: graphDurationMs },
      totalDurationMs: Date.now() - totalStart
    }
  }

  /**
   * Read each index surface's self-reported outstanding maintenance work —
   * the observability seam so an operator sees a grind coming (rising
   * pending bytes/items, a stalled background pass) instead of discovering
   * it as a CPU storm or a transaction blowing its budget mid-flight (see
   * {@link TransactionTimeoutError}).
   *
   * PURE PASSTHROUGH: for each of vector/metadata/graph, this calls ONLY the
   * ACTIVE provider's own `maintenanceDebt?()` hook (the same per-surface
   * provider resolution {@link Brainy.warm} uses) and reports its
   * {@link ProviderMaintenanceDebt} payload verbatim. There is no JS-side
   * fallback computation, no threshold evaluation, and no polling — brainy
   * surfaces the truth the provider measured; the provider owns the numbers
   * and the operator owns the policy (what threshold matters, what action to
   * take). A surface whose active provider does not implement the hook
   * reports `'unavailable'` — never a guessed or zeroed payload.
   *
   * @returns A {@link MaintenanceDebtReport}: per-surface outcome + payload.
   * @example
   * ```typescript
   * const debt = await brain.maintenanceDebt()
   * if (debt.metadata.outcome === 'reported' && debt.metadata.debt?.pendingBytes) {
   *   console.log('metadata pending bytes:', debt.metadata.debt.pendingBytes)
   * }
   * ```
   */
  async maintenanceDebt(): Promise<MaintenanceDebtReport> {
    await this.ensureInitialized({ needs: ['vector', 'metadata', 'graph'] })

    // --- Vector ---------------------------------------------------------
    const vectorProvider = this.index as VectorIndexProvider & {
      maintenanceDebt?: () => Promise<ProviderMaintenanceDebt>
    }
    const vector =
      typeof vectorProvider.maintenanceDebt === 'function'
        ? { outcome: 'reported' as const, debt: await vectorProvider.maintenanceDebt() }
        : { outcome: 'unavailable' as const }

    // --- Metadata --------------------------------------------------------
    const metadataProvider = this.metadataIndex as unknown as MetadataIndexProvider
    const metadata =
      typeof metadataProvider.maintenanceDebt === 'function'
        ? { outcome: 'reported' as const, debt: await metadataProvider.maintenanceDebt() }
        : { outcome: 'unavailable' as const }

    // --- Graph -------------------------------------------------------------
    const graphProvider = this.graphIndex as GraphIndexProvider & {
      maintenanceDebt?: () => Promise<ProviderMaintenanceDebt>
    }
    const graph =
      typeof graphProvider.maintenanceDebt === 'function'
        ? { outcome: 'reported' as const, debt: await graphProvider.maintenanceDebt() }
        : { outcome: 'unavailable' as const }

    return { vector, metadata, graph }
  }

  /**
   * Explicitly warm up the embedding engine
   *
   * Use this to pre-initialize the Candle WASM embedding engine before
   * processing requests. The WASM module (93MB with embedded model) takes
   * 90-140 seconds to compile on throttled CPU environments like Cloud Run.
   *
   * Calling this during container startup ensures the first real request
   * doesn't pay the compilation cost.
   *
   * @example
   * ```typescript
   * // Option 1: Use eagerEmbeddings config (automatic during init)
   * const brain = new Brainy({ eagerEmbeddings: true })
   * await brain.init() // Embedding engine initialized here
   *
   * // Option 2: Manual warmup (more control)
   * const brain = new Brainy()
   * await brain.init()
   * await brain.warmupEmbeddings() // Explicit control over timing
   * ```
   *
   * @returns Promise that resolves when embedding engine is ready
   */
  async warmupEmbeddings(): Promise<void> {
    if (!this.initialized) {
      throw new Error('Brain must be initialized before warming up embeddings. Call init() first.')
    }

    // Plugin-provided embeddings are already ready (native, no WASM warmup needed)
    if (this.pluginRegistry.hasProvider('embeddings')) {
      return
    }

    console.log('Warming up embedding engine...')
    const start = Date.now()
    await embeddingManager.init()
    const elapsed = Date.now() - start
    console.log(`Embedding engine ready in ${elapsed}ms`)
  }

  /**
   * Check if embedding engine is initialized
   *
   * @returns true if embedding engine is ready for immediate use
   */
  isEmbeddingReady(): boolean {
    // Plugin-provided embeddings are always ready (native, no WASM init required)
    if (this.pluginRegistry.hasProvider('embeddings')) {
      return true
    }
    return embeddingManager.isInitialized()
  }

  /**
   * Whether the process-global WASM embedding engine (all-MiniLM-L6-v2,
   * fixed 384-dim output, ≈93MB with the bundled model, 90-140s cold compile
   * on throttled CPUs) is this instance's active embedder — `false` when a
   * plugin has replaced it via the `'embeddings'` provider key. A native
   * provider has no such cold-start cost and may use a different output
   * dimension, so it is never worth avoiding.
   *
   * Used by init-path bootstrap writes (the VFS root — see
   * `VirtualFileSystem.doInitializeRoot()`) to decide whether embedding a
   * value during `init()` risks paying the WASM engine's cold compile.
   *
   * @returns true when the default WASM engine is active (no native
   *   `'embeddings'` provider registered).
   */
  usesDefaultWasmEmbedder(): boolean {
    return !this.pluginRegistry.hasProvider('embeddings')
  }

  /**
   * @description LEG C of the zero-norm/unvector-door law — migrate a
   * legacy zero-norm VFS root BEFORE the vector-leg open gate
   * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever
   * compares the canonical vectored-noun count against the vector index's
   * size. A pre-fix store may have persisted the VFS root (the fixed
   * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside
   * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance,
   * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt
   * refuses to insert a zero-norm vector) and never meant to cross an
   * engine boundary. Left unmigrated, the canonical ledger still counts it
   * as vectored while the vector index correctly holds nothing for it — a
   * near-empty store whose ONLY vectored row is this zero-norm root reads
   * "canonical vectored 1, index size 0" and throws
   * `VectorIndexNotReadyError` at open, going DARK instead of serving.
   *
   * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step
   * is safe under that law BECAUSE the VFS root lives at a FIXED,
   * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors
   * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same
   * convention as the other reserved-root literals in this file and in
   * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import
   * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE
   * direct canonical read by id (`storage.getNoun`, the same O(1)
   * fixed-path lookup {@link unvectorNounForRootMigration} itself uses
   * internally), NEVER a listing or a walk over `entities/nouns/**`. An
   * absent root (a store that has never used the VFS) is a no-op, no error.
   *
   * Runs UNCONDITIONALLY at every open, independent of whether a
   * `VirtualFileSystem` is ever constructed this session — the vector-leg
   * gate this fixes runs during Brainy's OWN init, before any
   * `VirtualFileSystem` instance exists to run its own lazy migration at
   * `doInitializeRoot()` (kept in place as the second line of defense for a
   * VFS actually opened this session — belt AND suspenders, never either
   * alone).
   */
  private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise<void> {
    const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
    // TORN-TOLERANT: a torn root record is a recovery-walk healer's job
    // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time
    // walk that meets a torn record narrates+counts, via the adapter's own
    // loud floor at the read site, and heals PAST it; the open itself must
    // still succeed), not this O(1) migration check's. Skip this open's
    // migration attempt rather than aborting init(): this leg is a
    // defensive EXTRA (the index belt + VirtualFileSystem's own
    // doInitializeRoot() migration still stand as the other lines of
    // defense), and it retries harmlessly at a later open once the root
    // heals.
    let root: HNSWNounWithMetadata | null
    try {
      root = await this.storage.getNoun(VFS_ROOT_ID)
    } catch (err) {
      if ((err as { code?: string }).code !== 'TORN_RECORD') throw err
      prodLog.warn(
        `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` +
          `migration check this open (the recovery walk is the healer; this migration ` +
          `retries harmlessly once the root heals).`
      )
      return
    }
    if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return
    if (!isZeroNormVector(root.vector)) return
    const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID)
    if (migrated) {
      prodLog.warn(
        `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` +
          `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` +
          `before the vector-leg open gate compares canonical-vectored-count against the ` +
          `vector index, so a near-empty store never reads a false coverage gap.`
      )
    }
  }

  /**
   * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's
   * persisted vector from a real (non-empty) vector to the "unvectored"
   * empty-array shape: the vector record is rewritten to `[]`, the row is
   * removed from the vector index (if present), and the vectored-noun
   * ledger (`getCanonicalCounts().vectors.all`) is decremented through the
   * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the
   * coverage ledger never silently drifts.
   *
   * Exists SOLELY for the VFS root zero-norm migration, called from two
   * sites that detect the same legacy shape (a persisted root whose vector
   * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded}
   * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg
   * C of the zero-norm/unvector-door law) and
   * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for
   * a VFS actually constructed this session). This is NOT the general-
   * purpose unvector API — ordinary application data uses the sanctioned
   * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside
   * `transact()`), which decrements the ledger and clears any pending
   * deferred-embed marker inline; it does not call this method. Never call
   * this outside a VFS root migration.
   *
   * Idempotent: a noun already unvectored (`vector.length === 0`) or absent
   * is a no-op — safe to call on every `init()`.
   *
   * @param id - The canonical noun id to migrate.
   * @returns `true` if a migration write happened, `false` if the noun was
   *   already unvectored (or absent) — a no-op.
   */
  async unvectorNounForRootMigration(id: string): Promise<boolean> {
    const noun = await this.storage.getNoun(id)
    if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false

    await this.persistSingleOp({ nouns: [id] }, async (tx) => {
      // Rewrite the vector leg to the unvectored shape. Placeholder adjacency
      // (mirrors update()'s own SaveNounOperation staging) — the op preserves
      // stored graph state when `connections.size === 0`.
      tx.addOperation(
        new SaveNounOperation(this.storage, {
          id,
          vector: [],
          connections: new Map(),
          level: 0
        })
      )
      // Remove from the vector index — safe even if the row was never
      // actually indexed (RemoveFromVectorIndexOperation's removeItem is a
      // no-op when the id is absent).
      tx.addOperation(
        new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)
      )
    })

    // Vectored-noun ledger: this migration carries a vector write with no
    // accompanying metadata operation (metadata is untouched), so the
    // saveNounMetadata(..., hasVector) seam never fires for it — mirrors the
    // deferred-embed LANDING path's use of the narrow storage hook, in
    // reverse.
    await this.storage.noteVectorUnlanded?.(id)
    return true
  }

  /**
   * Setup embedder
   */
  private setupEmbedder(): EmbeddingFunction {
    // Custom model loading removed - not implemented
    // Only 'fast' and 'accurate' model types are supported
    return defaultEmbeddingFunction
  }

  /**
   * Setup storage
   */
  /**
   * @description 7.x → 8.0 on-disk LAYOUT migration, run BEFORE `setupStorage()`.
   *
   * 7.x stored every object branch-scoped under `branches/<branch>/<basePath>`;
   * 8.0 stores the IDENTICAL entity structure at the storage ROOT
   * (`entities/<kind>/<shard>/<id>/…`) plus the generational `_system` +
   * `_generations` layer. `GenerationStore.open()` is tolerant of a missing
   * manifest (opens at generation 0), so a naive 8.0 open of a 7.x directory
   * reports zero entities and SILENTLY LOSES ALL DATA. This phase collapses the
   * HEAD branch's canonical entities to the root so the rest of init (and any
   * native plugin storage factory, whose own legacy guard would otherwise THROW)
   * sees a clean flat-v8 store; 8.0 rebuilds all derived state from those entities.
   *
   * Runs on a temporary BUILT-IN `FileSystemStorage` (never the plugin adapter),
   * is idempotent (a `_system/migration-layout.json` marker short-circuits re-open),
   * resume-safe (per-object read→write→delete), and a strict NO-OP for any brain
   * that is not a filesystem store carrying a legacy `branches/` layout.
   */
  private async legacyLayoutMigrationPhase(): Promise<void> {
    // Pre-built adapter instances and non-filesystem stores have no on-disk 7.x
    // layout to migrate.
    if (isStorageAdapterInstance(this.config.storage)) return
    const storageConfig = (this.config.storage || {}) as StorageOptions & Record<string, unknown>
    const type = storageConfig.type || 'auto'
    if (type !== 'filesystem' && type !== 'auto') return

    // Fast path: the only on-disk shape that needs migrating has a top-level
    // `branches/` directory. A native 8.0 brain and a fresh directory do not, so
    // skip the probe-storage construction entirely on the init hot path. (The
    // migration is node-filesystem-only; `fs.existsSync` is absent/throwing
    // elsewhere, which the catch treats as "nothing to migrate".)
    //
    // Resolve the root through the SHARED resolver so the migration probe checks
    // the SAME directory createStorage (and any plugin factory) will open, and a
    // removed pre-8.0 alias (`rootDirectory`, `options.*`, `fileSystemStorage.*`)
    // throws here too — consistent with createStorage, never a silent skip.
    const rootDir = resolveFilesystemRoot(storageConfig)
    try {
      if (!fs.existsSync(`${rootDir}/branches`)) return
    } catch {
      return
    }

    // Probe with a BUILT-IN filesystem storage over the same root (never the
    // plugin factory — cor's adapter would throw on the legacy layout). In a
    // non-filesystem environment this throws; that just means "nothing to migrate".
    let probe: BaseStorage
    try {
      probe = (await createStorage({ ...storageConfig, type: 'filesystem' })) as BaseStorage
      await probe.init()
    } catch {
      return
    }

    try {
      // Idempotency: a completed migration leaves a marker → re-open is a no-op.
      const marker = await probe.readRawObject('_system/migration-layout.json')
      if (marker && (marker as { layout?: string }).layout === 'flat-v8') return

      // DETECT: HEAD-branch entities present under branches/<head>/entities/.
      const head = (storageConfig.branch as string) || 'main'
      const branchPaths = await probe.listRawObjects('branches')
      const headEntityPrefix = `branches/${head}/entities/`
      const legacyEntityPaths = branchPaths.filter((p) => p.startsWith(headEntityPrefix))

      if (legacyEntityPaths.length === 0) {
        // Already flat (root entities, no head-branch entities) → stamp the marker
        // so future opens short-circuit. A genuinely empty/fresh dir gets no marker.
        // "Are there any entities?" is answered by ONE directory read, not by a
        // recursive listing of every file in the tree: this runs on the open path
        // of every store that does not yet carry the marker (a restore, a store
        // built by an older release), and on a large store that listing walks the
        // whole canonical tree to learn a boolean.
        const oneLevel = (
          probe as unknown as { listRawPrefixes?: (prefix: string) => Promise<string[]> }
        ).listRawPrefixes
        const hasRootEntities =
          typeof oneLevel === 'function'
            ? (await oneLevel.call(probe, 'entities')).length > 0
            : (await probe.listRawObjects('entities')).length > 0
        if (hasRootEntities) {
          await probe.writeRawObject('_system/migration-layout.json', {
            layout: 'flat-v8',
            version: 8,
            fromBranch: null,
            entitiesMoved: 0
          })
        }
        return
      }

      // GUARD: an explicit autoMigrate:false must not silently lose 7.x data.
      if (this.config.autoMigrate === false) {
        throw new Error(
          `Brainy 8.0 found a 7.x branch layout at this storage directory ` +
            `(${legacyEntityPaths.length} entities under branches/${head}/). Opening it as-is ` +
            `would report zero entities and lose your data. Set autoMigrate: true (the default) ` +
            `to migrate the layout in place on open, or export the data on 7.x first.`
        )
      }

      // Acquire the writer lock for the duration of the move (idempotent marker
      // makes a lost race harmless, but the lock prevents two concurrent collapses).
      const canLock =
        typeof (probe as unknown as { supportsMultiProcessLocking?: () => boolean })
          .supportsMultiProcessLocking === 'function' &&
        (probe as unknown as { supportsMultiProcessLocking: () => boolean }).supportsMultiProcessLocking()
      const lockable = probe as unknown as {
        acquireWriterLock?: (o: { force?: boolean }) => Promise<void>
        releaseWriterLock?: () => Promise<void>
      }
      if (canLock && typeof lockable.acquireWriterLock === 'function') {
        await lockable.acquireWriterLock({ force: this.config.force })
      }

      if (!this.config.silent) {
        console.warn(
          `[brainy] Migrating a 7.x branch layout (branches/${head}) to the 8.0 flat layout ` +
            `in place — ${legacyEntityPaths.length} entity files. This runs once; back up the ` +
            `directory first if you need a rollback (8.0 does not keep the old layout).`
        )
      }

      try {
        // COLLAPSE: branches/<head>/entities/* → entities/* via the .gz-transparent
        // raw primitives (NOT fs.rename — listRawObjects returns logical paths and
        // the in-memory adapter has no real files). read→write→delete is idempotent,
        // so a crash mid-move is recovered by simply re-running.
        let moved = 0
        for (const src of legacyEntityPaths) {
          const target = src.slice(`branches/${head}/`.length) // → entities/...
          const data = await probe.readRawObject(src)
          if (data === null) continue // already moved by an interrupted prior run
          await probe.writeRawObject(target, data)
          await probe.deleteRawObject(src)
          moved++
        }

        // REBUILD persisted derived state from the now-flat canonical entities.
        // (The three indexes — HNSW, metadata, graph — are rebuilt by the normal
        // init's rebuildIndexesIfNeeded after storage is set up; the COUNT rollups
        // are NOT, so they must be rebuilt + persisted here or getNounCount()/stats()
        // read 0 on the migrated store.)
        //   - rebuildCounts → totalNounCount/totalVerbCount + counts.json (getNounCount/getVerbCount)
        //   - rebuildTypeCounts → per-type type-statistics.json (stats().entitiesByType)
        await rebuildCounts(probe)
        await probe.rebuildTypeCounts()
        if (typeof (probe as unknown as { rebuildSubtypeCounts?: () => Promise<void> }).rebuildSubtypeCounts === 'function') {
          await (probe as unknown as { rebuildSubtypeCounts: () => Promise<void> }).rebuildSubtypeCounts()
        }

        // FINALIZE: mark migrated (makes re-open a no-op), then drain the head
        // branch. Non-head branches under branches/ are 7.x version history 8.0's
        // MVCC does not import — they are left in place.
        await probe.writeRawObject('_system/migration-layout.json', {
          layout: 'flat-v8',
          version: 8,
          fromBranch: head,
          entitiesMoved: moved
        })

        // Parity guard (defense-in-depth; the VFS `_cow/` stranding lesson). The
        // entity move deleted every `branches/<head>/entities/*` key, so anything
        // STILL under `branches/<head>/` is non-entity durable state a 7.x engine
        // wrote outside `entities/` (branch-scoped blobs, an index dir, a field
        // registry). Draining it unconditionally would silently DELETE it — the
        // same failure class as the VFS content blobs stranded in `_cow/`. So
        // drain only a clean branch (nothing but the moved entities); if any
        // non-entity object survives, PRESERVE the branch (skip the drain) so the
        // state stays recoverable, and surface it loudly.
        const residual = (
          await probe.listRawObjects(`branches/${head}`)
        ).filter((k: string) => !k.startsWith(`branches/${head}/entities/`))
        if (residual.length === 0) {
          await probe.removeRawPrefix(`branches/${head}`)
        } else if (!this.config.silent) {
          const sample = residual.slice(0, 8).join(', ')
          console.warn(
            `[brainy] 7→8 migration preserved ${residual.length} non-entity object(s) under ` +
              `branches/${head}/ — branch-scoped durable state written outside entities/. The ` +
              `branch was NOT drained, so this state is recoverable; inspect it before removing ` +
              `branches/ manually. Keys: ${sample}${residual.length > 8 ? ', …' : ''}`
          )
        }
      } finally {
        if (canLock && typeof lockable.releaseWriterLock === 'function') {
          await lockable.releaseWriterLock()
        }
      }
    } finally {
      if (typeof (probe as unknown as { close?: () => Promise<void> }).close === 'function') {
        await (probe as unknown as { close: () => Promise<void> }).close()
      }
    }
  }

  /**
   * @description On-open recovery for VFS content blobs a 7→8 upgrade left in the
   * removed branch system's copy-on-write area (`_cow/`). If a `_cow/` area is
   * present and a completed adoption has not already been recorded, adopt every
   * orphaned blob into the 8.0 content-addressed store (`_cas/`) and stamp a
   * marker so later opens skip the scan. Because it runs on every open right
   * after {@link legacyLayoutMigrationPhase}, it heals BOTH a fresh 7→8 migration
   * (where `_cow/` still holds the blobs) AND a brain already migrated by an
   * earlier 8.0.x that stranded them — no operator action needed.
   *
   * Filesystem-only; a native-8.0 / fresh brain has no `_cow/` and returns on the
   * cheap existence check. Non-destructive and idempotent (see
   * {@link BaseStorage.adoptLegacyCowBlobs}). If any blob cannot be fully adopted
   * (`incomplete > 0`), the marker is NOT stamped (the next open retries) and
   * {@link _vfsBlobAdoptionIncomplete} is set so the pre-upgrade backup is
   * retained — the upgrade is not verifiably complete for the VFS.
   */
  private async autoAdoptLegacyVfsBlobsIfNeeded(): Promise<void> {
    if (this.getStorageType() !== 'filesystem') return
    // Cheapest gate: no copy-on-write area → a native-8.0 / fresh brain, nothing
    // a 7.x brain could have stranded here.
    try {
      const root = resolveFilesystemRoot(
        (this.config.storage || {}) as StorageOptions & Record<string, unknown>
      )
      if (!fs.existsSync(`${root}/_cow`)) return
    } catch {
      return
    }

    const storage = this.storage as unknown as {
      adoptLegacyCowBlobs?: () => Promise<{
        cowBlobs: number
        adopted: number
        alreadyPresent: number
        incomplete: number
      }>
      readRawObject?: (p: string) => Promise<unknown>
      writeRawObject?: (p: string, o: unknown) => Promise<void>
    }
    if (typeof storage.adoptLegacyCowBlobs !== 'function') return

    const MARKER = '_system/vfs-blob-adoption.json'
    try {
      const done = (await storage.readRawObject?.(MARKER)) as { complete?: boolean } | null
      if (done && done.complete) return // already fully adopted on a prior open
    } catch {
      // no marker yet — proceed
    }

    const result = await storage.adoptLegacyCowBlobs()

    if (result.incomplete > 0) {
      // Partial: retry on future opens, and hold the pre-upgrade backup.
      this._vfsBlobAdoptionIncomplete = true
      if (!this.config.silent) {
        console.error(
          `[brainy] VFS blob recovery adopted ${result.adopted} blob(s) but ${result.incomplete} ` +
            `orphaned _cow/ blob(s) are missing their bytes or metadata and were left in place. ` +
            `The pre-upgrade backup is being retained; inspect _cow/ before discarding it.`
        )
      }
      return
    }

    // Clean sweep — record it so later opens skip the _cow/ scan entirely.
    try {
      await storage.writeRawObject?.(MARKER, { complete: true, adopted: result.adopted })
    } catch {
      // marker write is best-effort; adoption already succeeded
    }
    if (result.adopted > 0 && !this.config.silent) {
      console.warn(
        `[brainy] Recovered ${result.adopted} VFS content blob(s) stranded by a 7→8 upgrade ` +
          `(adopted _cow/ → _cas/ in place).`
      )
    }
  }

  private async setupStorage(): Promise<BaseStorage> {
    // If the caller passed a pre-constructed storage adapter (e.g.
    // `storage: new MemoryStorage()`, or the historical materializer's
    // pre-populated snapshot storage), use it directly instead of routing
    // through the factory. Otherwise the factory's `type === 'auto'` branch
    // would silently create a FileSystemStorage at `./brainy-data`, ignoring
    // the instance and surprising anyone who wrote `new MemoryStorage()`
    // expecting it to be honoured.
    if (isStorageAdapterInstance(this.config.storage)) {
      return this.config.storage as BaseStorage
    }

    // The config-object branch of `BrainyConfig['storage']` is a structural
    // subset of the factory's `StorageOptions`; the Record view serves the
    // plugin factory contract (`create(config: Record<string, unknown>)`).
    const storageConfig = (this.config.storage || {}) as StorageOptions &
      Record<string, unknown>
    const storageType = storageConfig.type || 'auto'

    // Check plugin-provided storage factories (e.g., a native filesystem/mmap
    // override registered by an acceleration plugin).
    const pluginFactory = this.pluginRegistry.getStorageFactory(storageType)
    if (pluginFactory) {
      // Hand the plugin factory a NORMALIZED config whose canonical `path` is
      // the SAME root our built-in resolver computed. The plugin's native side
      // (e.g. getBinaryBlobPath / mmap files) re-resolves the directory itself;
      // without this, an alias-only config (`{ rootDirectory }`, `{ options:
      // { path } }`, …) would resolve to the alias here but to ./brainy-data on
      // the plugin side — a brainy-data/cor split-brain where the two halves
      // write to different directories. Stamping the resolved `path` keeps both
      // resolvers on the identical root.
      const normalizedConfig = {
        ...storageConfig,
        path: resolveFilesystemRoot(storageConfig)
      }
      const adapter = await pluginFactory.create(normalizedConfig)
      return adapter as BaseStorage
    }

    // Fall through to built-in storage types
    const storage = await createStorage(storageConfig)
    return storage as BaseStorage
  }

  /**
   * Detect storage type from the storage instance class name
   *
   * Fixes storage type detection for HNSW persistence mode.
   * Previously relied on this.config.storage.type which was often not set
   * after storage creation, causing cloud storage to use 'immediate' mode
   * and resulting in 50-100x slower add() operations.
   *
   * @returns Storage type string ('gcs', 's3', 'memory', etc.)
   */
  private getStorageType(): string {
    if (!this.storage) return 'memory'

    const className = this.storage.constructor.name
    if (className.includes('Gcs') || className.includes('GCS')) return 'gcs'
    if (className.includes('S3')) return 's3'
    if (className.includes('R2')) return 'r2'
    if (className.includes('Azure')) return 'azure'
    if (className.includes('OPFS')) return 'opfs'
    if (className.includes('FileSystem')) return 'filesystem'
    if (className.includes('Memory')) return 'memory'

    return 'unknown'
  }

  /**
   * Setup index — single unified vector graph.
   *
   * Persistence mode is auto-selected from the storage adapter (see
   * {@link resolveHNSWPersistMode}): `'immediate'` on filesystem storage,
   * `'deferred'` on memory storage. Operators can override with an explicit
   * `config.vector.persistMode` (e.g. `'deferred'` for bulk-ingest speed on
   * filesystem storage).
   */
  private setupIndex(): JsHnswVectorIndex {
    // 8.0 config surface: config.vector.{recall, persistMode}.
    // The recall preset translates to HNSW knobs (M / efConstruction / efSearch)
    // via resolveJsHnswConfig. No algorithm-internal knobs are exposed at the
    // public surface. The JS index computes exact float32 distances throughout —
    // there is no quantization path.
    const recallKnobs = resolveJsHnswConfig(this.config.vector)
    const indexConfig = {
      ...recallKnobs,
      distanceFunction: this.distance
    }

    const persistMode = this.resolveHNSWPersistMode()

    return new JsHnswVectorIndex(indexConfig, this.distance, {
      storage: this.storage,
      useParallelization: true,
      persistMode
    })
  }

  /**
   * Create the vector index, using the plugin-provided engine when available.
   * Shared by init() and clear() to avoid duplication.
   *
   * Selection order (first match wins):
   * 1. `'vector'` provider if registered — the canonical 8.0 provider key.
   *    A native plugin registers one internally-adaptive engine here;
   *    in-memory / hybrid / on-disk selection is the provider's job, not
   *    Brainy's, so there is no engine-specific branching on this side.
   * 2. Brainy's built-in TS JsHnswVectorIndex (the always-available fallback).
   */
  private createIndex(): JsHnswVectorIndex {
    const persistMode = this.resolveHNSWPersistMode()
    const vectorFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('vector')
    if (vectorFactory) {
      // Native providers receive the algorithm-neutral surface: the resolved
      // recall preset name (per the 8.0 provider contract they translate it
      // to their own internal knobs), plus the JS HNSW knob tuple for
      // providers that mirror the open-core implementation. Quantization at
      // scale is the native provider's own concern (e.g. DiskANN PQ) — Brainy
      // exposes no quantization knob.
      return vectorFactory(
        {
          ...resolveJsHnswConfig(this.config.vector),
          recall: this.config.vector?.recall ?? DEFAULT_RECALL,
          distanceFunction: this.distance
        },
        this.distance,
        { storage: this.storage, persistMode }
      )
    }
    return this.setupIndex()
  }

  /**
   * Resolve the vector persistence mode from the storage adapter.
   *
   * Auto-selected when `config.vector.persistMode` is omitted:
   * - **filesystem** (and any persistent adapter): `'immediate'` — per-add
   *   durability is the point of a persistent backend.
   * - **memory**: `'deferred'` — nothing survives the process anyway, so
   *   per-add persistence writes are pure overhead; deferred batches them
   *   into `flush()` / `close()`.
   *
   * An explicit `config.vector.persistMode` always wins (the operator
   * escape hatch for bulk-ingest pipelines on filesystem storage).
   */
  private resolveHNSWPersistMode(): 'immediate' | 'deferred' {
    if (this.config.vector?.persistMode) {
      return this.config.vector.persistMode
    }
    return this.getStorageType() === 'memory' ? 'deferred' : 'immediate'
  }

  /**
   * Normalize and validate configuration
   */
  private normalizeConfig(config?: BrainyConfig): ResolvedBrainyConfig {
    // Validate storage configuration. Brainy 8.0 ships two adapters only —
    // FileSystemStorage and MemoryStorage (cloud + OPFS adapters were removed).
    // Cloud backup remains supported via operator tooling (db.persist() +
    // gsutil / aws s3 cp / rclone / azcopy). Pre-constructed adapter
    // instances bypass the type check (they ARE the storage).
    const storageConfig = config?.storage
    if (
      storageConfig &&
      !isStorageAdapterInstance(storageConfig) &&
      storageConfig.type &&
      !['auto', 'memory', 'filesystem'].includes(storageConfig.type)
    ) {
      throw new Error(
        `Invalid storage type: ${storageConfig.type}. Brainy 8.0 ships 'auto', 'memory', and 'filesystem' only. ` +
          `Cloud storage adapters (GCS / S3 / R2 / Azure) and OPFS were removed in 8.0; ` +
          `back up locally with db.persist() and sync the on-disk artefact with your tool of choice.`
      )
    }

    return {
      storage: config?.storage || { type: 'auto' },
      verbose: config?.verbose ?? false,
      silent: config?.silent ?? false,
      // false = auto-decide based on dataset size (inline vs lazy rebuild)
      disableAutoRebuild: config?.disableAutoRebuild ?? false,
      // Memory management escape hatches over the RAM-derived auto limits
      maxQueryLimit: config?.maxQueryLimit ?? undefined,
      reservedQueryMemory: config?.reservedQueryMemory ?? undefined,
      // Migration LOCK wait budget — left undefined when omitted so
      // awaitMigrationLock() applies its 30 s default at the read site.
      migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined,
      // Apply-phase transaction budget floor — left undefined when omitted so
      // transactTimeoutBudget() applies its 30 s default at the read site.
      transactionBudgetFloorMs: config?.transactionBudgetFloorMs ?? undefined,
      // Pre-upgrade backup — default-on; opt out with `migrationBackup: false`.
      migrationBackup: config?.migrationBackup ?? true,
      // Vector index configuration (8.0) — algorithm-neutral surface with
      // `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode).
      // See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2.
      vector: config?.vector ?? undefined,
      // Generational-history retention — the `retention` knob; defaults applied
      // at the read site (resolveRetentionPolicy) so a partial object inherits
      // per-field defaults and `undefined` resolves to ADAPTIVE.
      retention: config?.retention ?? undefined,
      // Embedding initialization — left undefined when omitted so the adaptive
      // default resolves at the init() read site (eager when the WASM embedder
      // is the active one on a non-reader writer outside unit tests). Explicit
      // true/false always wins. See the eagerEmbeddings JSDoc in brainy.types.ts.
      eagerEmbeddings: config?.eagerEmbeddings ?? undefined,
      // Eager index warm at init() — default off (lazy demand-load), the
      // pre-8.9 behavior. See the warmOnOpen JSDoc in brainy.types.ts.
      warmOnOpen: config?.warmOnOpen ?? false,
      // Plugin configuration - undefined = auto-detect
      plugins: config?.plugins ?? undefined,
      // Integration Hub - undefined/false = disabled
      integrations: config?.integrations ?? undefined,
      // Migration — auto-run by default. Pending data migrations apply inline
      // during init() for small datasets (<10K entities); larger datasets
      // defer with a notice so the operator can schedule brain.migrate().
      autoMigrate: config?.autoMigrate ?? true,
      // Subtype required-by-default (8.0 — per BRAINY-8.0-SUBTYPE-CONTRACT § C-1).
      // Every write path now requires `subtype` on every type unless the consumer
      // opts out explicitly. Opt-out: `requireSubtype: false` (last-resort migration
      // hatch for old data or legacy fixtures) or `requireSubtype: { except: [...] }`
      // (per-type allowlist).
      requireSubtype: config?.requireSubtype ?? true,
      // Multi-process safety
      mode: config?.mode ?? 'writer',
      force: config?.force ?? false,
      // Engine-owned persistence cadence — defaults resolve at the trigger
      // site (policy 'auto': 512 writes / 30s interval / 2s idle).
      persistence: config?.persistence,
      logAuthority: config?.logAuthority ?? 'adopt'
    }
  }

  /**
   * @description THE READ GATE. Every read choke point (getNeighborUuids,
   * find, filterIdsBelted, getTypedNeighbors) calls this before touching a
   * derived index. It is a CHECK, never a build: it asks each of the three
   * providers (vector, metadata, graph) for its named health verdict via
   * {@link assessProviderHealth} — the provider's own sync, O(1)
   * `healthReport()` when exposed, else the `isReady()` / size-heuristic
   * fallback — and either lets the read proceed or throws the matching typed
   * `*NotReadyError` naming the provider and its failing reasons. It NEVER
   * triggers a rebuild and NEVER walks the store: a needed rebuild is
   * entirely open()'s job (see {@link rebuildIndexesIfNeeded}), which runs to
   * completion before `init()` returns — so by the time any read reaches
   * this gate, a healthy provider is already built. A migrating provider is
   * deferred to exactly as before (it owns its own in-place rebuild).
   *
   * A report with something worth telling an operator (a failing invariant,
   * whether serving or not, or a named `unledgered` family) narrates via
   * `prodLog.warn` ONCE per (provider, `report.generation`) — never once per
   * read — before any throw decision is made.
   */
  /**
   * @description Whether two entity `data` payloads are the same content —
   * the "no re-embed on unchanged data" comparison. Primitives compare by
   * value; objects compare structurally with key order normalized.
   * @param a - The incoming data.
   * @param b - The stored data.
   * @returns `true` when the content is identical.
   */
  private static sameEntityData(a: unknown, b: unknown): boolean {
    if (a === b) return true
    if (a === null || b === null || typeof a !== typeof b) return false
    if (typeof a !== 'object') return false
    const stable = (v: unknown): string =>
      JSON.stringify(v, (_k, val) =>
        val && typeof val === 'object' && !Array.isArray(val)
          ? Object.keys(val as Record<string, unknown>).sort().reduce((o, k) => {
              ;(o as Record<string, unknown>)[k] = (val as Record<string, unknown>)[k]
              return o
            }, {} as Record<string, unknown>)
          : val
      )
    try { return stable(a) === stable(b) } catch { return false }
  }

  private ensureIndexesLoaded(
    families: ReadonlyArray<'vector' | 'metadata' | 'graph'> = ['vector', 'metadata', 'graph']
  ): void {
    // PER-FAMILY SCOPE. This gate used to refuse on ANY provider's not-ready
    // verdict at every read choke point — so a pure metadata find({where})
    // was refused because the VECTOR leg was not serving; a production
    // deployment's badge reads returned 500s for exactly that reason on the
    // pair's first adoption. A read may only be refused by the family it
    // actually consults: metadata reads by the metadata leg (+ graph for a
    // `connected` filter), vector search by the vector leg, traversal by the
    // graph leg. Callers name what they need.
    const all: ReadonlyArray<readonly ['vector' | 'metadata' | 'graph', unknown, new (message: string, originalError?: Error) => BrainyError]> = [
      ['vector', this.index, VectorIndexNotReadyError],
      ['metadata', this.metadataIndex, MetadataIndexNotReadyError],
      ['graph', this.graphIndex, GraphIndexNotReadyError]
    ]
    const providers = all.filter(([name]) => families.includes(name))

    for (const [name, provider, ErrorClass] of providers) {
      // Migration LOCK (#18) deference: a migrating provider owns its own
      // in-place rebuild — brainy must not judge (or race) it here.
      if (this.providerIsMigrating(provider)) continue

      const assessment = assessProviderHealth(provider)

      if (assessment.reasons.length > 0 && assessment.report != null) {
        const generation = assessment.report.generation
        // Dedupe by CONTENT, not by the provider's generation counter — see
        // _lastNarratedHealth. The generation is still REPORTED (an operator
        // wants to know which generation produced the verdict); it just no
        // longer decides whether the line is worth saying.
        const line =
          `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` +
          assessment.reasons.join('; ')
        const key = `${assessment.report.provider}\u0000${assessment.reasons.join('; ')}`
        if (this._lastNarratedHealth.get(provider) !== key) {
          this._lastNarratedHealth.set(provider, key)
          prodLog.warn(line)
        }
      }

      if (assessment.readiness === 'not-ready') {
        // A provider REBUILDING ITSELF gets a refusal that says so, with its
        // own progress: open deliberately did not wait for it, this door is
        // temporarily closed, and it opens by itself. Distinct from a broken
        // index, which needs an operator.
        const rebuilding = assessProviderRebuild(provider)
        if (rebuilding) {
          throw new ErrorClass(
            `${name} index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` +
              `Reads of this family refuse rather than serve an empty result. The brain is open ` +
              `and every other family is serving; this door opens by itself when the provider ` +
              `reports serving — no action is needed.`
          )
        }
        throw new ErrorClass(
          `${name} index is not serving (via ${assessment.via}): ` +
            `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` +
            `empty result — open() builds the derived indexes; a read never does. Rebuild via ` +
            `repairIndex({ rebuild: ['${name}'] }) or reopen the brain.`
        )
      }
    }
  }

  /**
   * @description Wire the shared UUID ↔ int resolver (the metadata index's
   * idMapper) into the JS graph index and the storage layer (8.0 u64
   * contract). Called after the graph index is resolved on init so every
   * consumer of the BigInt boundary shares one int
   * universe. Native graph-index providers carry their own mapper and don't
   * expose `setEntityIdMapper` — the storage-level wiring still applies so
   * its verb read paths can convert UUIDs before provider calls.
   */
  private wireGraphIdResolver(): void {
    const resolver = this.metadataIndex.getIdMapper()
    this.storage.setGraphEntityIdResolver(resolver)
    const jsGraphIndex = this.graphIndex as Partial<Pick<GraphAdjacencyIndex, 'setEntityIdMapper'>>
    if (typeof jsGraphIndex.setEntityIdMapper === 'function') {
      jsGraphIndex.setEntityIdMapper(resolver)
    }
  }

  /**
   * @description Wire the HNSW connections codec (2.4.0 #3). Activates when
   * BOTH (a) the `graph:compression` provider is registered (cor registers
   * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the
   * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW
   * keeps working via the legacy JSON-array path.
   *
   * This layer does NOT require a real local path — the binary-blob primitive
   * saves the compressed bytes through the storage adapter directly, so the
   * codec works on the memory adapter as well as filesystem. (A native vector
   * provider, by contrast, persists its own single `.dkann` file and has no
   * per-node connection lists, so it is skipped — see the feature-detect below.)
   * Format convergence is lazy: pre-2.4.0 nodes load via the legacy path,
   * then the next dirty save writes the compressed form (and the legacy
   * field of saveVectorIndexData becomes empty), so all reads converge over time
   * without an explicit migration step.
   */
  private wireConnectionsCodec(): void {
    const provider = this.pluginRegistry.getProvider<GraphCompressionProvider>('graph:compression')
    if (!provider) return

    const idMapper = this.metadataIndex.getIdMapper?.()
    if (!idMapper) return

    // Feature-detect: only the JS HNSW path uses per-node connection lists.
    // Native vector-index providers (DiskANN-style) persist the graph as a
    // single mmap'd file and have no analogue, so skip the wiring there.
    // Pre-8.0 the wire was unconditional and relied on the native wrapper
    // exposing a no-op setConnectionsCodec(); 8.0 makes it feature-detected.
    if (typeof (this.index as { setConnectionsCodec?: unknown }).setConnectionsCodec !== 'function') return

    const codec = new ConnectionsCodec(provider, idMapper)
    this.index.setConnectionsCodec(codec)
    if (!this.config.silent) {
      console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)')
    }
  }

  /**
   * @description Consume the JS metadata index's watermark verdict (see
   * {@link MetadataIndexManager.watermarkVerdict}) at open — the coordinator
   * half of the catchup wiring; {@link MetadataIndexManager.applyWatermarkCatchup}
   * is the mechanism half. Feature-detected to the JS manager only: a native
   * metadata-index provider consumes the same verdict door in its own train
   * (this method never touches the native-provider wrapper contract).
   *
   * Ordering: called from `performInit()` immediately after
   * `metadataIndex.init()` has computed the verdict against the generation
   * store's now-FINAL committed generation, and BEFORE `rebuildIndexesIfNeeded()`
   * (the open-time rebuild gate) or any read serves — so a caller can never
   * observe the pre-catchup state.
   *
   * @param alreadyRebuilt - `true` when crash recovery just rebuilt every
   *   index from canonical (rolled-back uncommitted transactions) — the
   *   verdict's prescribed action is redundant with what already ran (a
   *   fresh canonical walk supersedes any catchup fold or rescan), so it is
   *   skipped, narrated, rather than duplicating the work.
   */
  private async consumeMetadataWatermarkVerdict(alreadyRebuilt: boolean): Promise<void> {
    if (!(this.metadataIndex instanceof MetadataIndexManager)) return
    const verdict = this.metadataIndex.watermarkVerdict()
    if (verdict === null || verdict === 'adopt') return

    if (alreadyRebuilt) {
      prodLog.info(
        `[Brainy] metadata index watermark verdict '${verdict}' at open — skipped: crash ` +
          `recovery already rebuilt every index from canonical this open.`
      )
      return
    }

    const window = this.metadataIndex.watermarkGap()
    // A genuine first boot (no persisted artifact at all) verdicts 'rescan'
    // too — same as a real unverifiable artifact — but it is routine, not
    // alarming: narrate it at info level instead of warn (mirrors the
    // manager's own internal distinction in loadWatermarkVerdict()).
    const firstBoot = verdict === 'rescan' && !this.metadataIndex.watermarkArtifactPresent()
    const preNarrate = firstBoot ? prodLog.info.bind(prodLog) : prodLog.warn.bind(prodLog)
    preNarrate(
      verdict === 'catchup' && window
        ? `[Brainy] metadata index watermark verdict: CATCHUP — folding generations ` +
            `(${window.from}, ${window.to}] from the fact log before this open serves reads.`
        : firstBoot
          ? `[Brainy] metadata index watermark verdict: rescan (no persisted artifact — first ` +
              `boot; the rebuild below is a trivial no-op walk).`
          : `[Brainy] metadata index watermark verdict: RESCAN — the persisted artifact is ` +
              `unverifiable (unstamped, or ahead of the store's committed generation); ` +
              `forcing a full rebuild from canonical at open.`
    )

    const scan = window
      ? this.scanFacts({ fromGeneration: window.from + 1, toGeneration: window.to })
      : null
    const result = await this.metadataIndex.applyWatermarkCatchup(scan)

    if (result.action === 'rescan') {
      const postNarrate = firstBoot ? prodLog.debug.bind(prodLog) : prodLog.warn.bind(prodLog)
      postNarrate(
        `[Brainy] metadata index catchup demoted to a full rebuild` +
          `${result.reason ? ` — ${result.reason}` : ''}.`
      )
    } else if (result.action === 'caught-up') {
      prodLog.warn(
        `[Brainy] metadata index catchup complete: ${result.factsApplied} fact(s) folded ` +
          `(${result.nounsApplied} noun op(s), ${result.verbsApplied} verb op(s)) — index now ` +
          `reflects generation ${result.window?.to}.`
      )
    }
  }

  /**
   * @description B3 Deliverable 3 — THE ONLINE METADATA REBUILD.
   * `repairIndex()`'s ceremony door for the `'metadata'` family routes here
   * instead of calling `MetadataIndexManager.rebuild()` directly: build a
   * FRESH replacement manager BESIDE the live one (same storage, same
   * idMapper — identity is shared, never a second mapper), walk canonical
   * into it while every live write during the build ALSO mirrors there
   * (`MetadataIndexManager.beginShadow`), fold the generation window the
   * walk may have read stale, then atomically swap this brain's reference —
   * `this.metadataIndex` points at the OLD manager for the ENTIRE build, so
   * every read in progress (and every read that starts before the swap
   * line executes) keeps serving its full, unbuilt-adjacent population;
   * nothing ever observes a half-built index.
   *
   * PERSISTENCE CHOICE (named per the B3 brief): the JS manager's persisted
   * keys (field-index chunks, column-store segments, the watermark stamp,
   * the id-mapper record) are GLOBAL per storage — not namespaced per
   * manager instance — so two managers cannot safely persist independently
   * mid-build (a segment-number race, a stamp race, an id-mapper reload
   * that would discard the live manager's not-yet-flushed assignments —
   * see `MetadataIndexManager.initForShadowBuild`'s JSDoc for the id-mapper
   * hazard specifically). This build therefore PERSISTS ONLY AT SWAP: the
   * shadow builds entirely in memory (`rebuild({ inMemoryOnly: true })` +
   * a fact-log fold — neither touches storage) and flushes exactly once,
   * after the swap, as the sole owner of the shared keys.
   *
   * FALLBACK: a store with no fact log (or a non-JS/native metadata
   * provider — its own train owns its online-rebuild strategy) cannot
   * safely bound "what landed during the walk"; this method falls back to
   * the ORIGINAL blocking clear-then-walk `rebuild()`, narrated.
   */
  private async rebuildMetadataIndexOnline(): Promise<void> {
    if (!(this.metadataIndex instanceof MetadataIndexManager)) {
      // A registered provider (e.g. a native accelerator) may replace
      // `this.metadataIndex` with a non-MetadataIndexManager object at
      // runtime even though the field's declared type is the JS class —
      // the cast mirrors the same reach-in used elsewhere in this file
      // (e.g. checkHealth()'s `metadataProvider` locals) for exactly this.
      const provider = this.metadataIndex as unknown as MetadataIndexProvider
      await provider.rebuild()
      return
    }

    const committedAtStart = this.storage.committedGeneration?.() ?? null
    const factLogAvailable = committedAtStart !== null && this.scanFacts() !== null
    if (!factLogAvailable) {
      prodLog.warn(
        `[Brainy] repairIndex(): metadata rebuild — no fact log on this store, build-beside ` +
          `is unavailable; falling back to the blocking rebuild (reads may serve a ` +
          `partially-built index for its duration).`
      )
      await this.metadataIndex.rebuild()
      return
    }

    prodLog.warn(
      `[Brainy] repairIndex(): metadata rebuild — building a fresh replacement index BESIDE ` +
        `the live one (reads keep serving the current index throughout); swapping in ` +
        `atomically once it is caught up.`
    )
    const startedAt = Date.now()
    const oldManager = this.metadataIndex
    const shadow = new MetadataIndexManager(this.storage, {}, {
      entityIdMapper: oldManager.getIdMapper()
    })

    oldManager.beginShadow(shadow)
    let committedAtSwap: number
    try {
      await shadow.buildBeside(committedAtStart!)
      // Capture the true final generation right before the swap — a
      // synchronous read, no `await` between here and the reference
      // assignment below, so nothing can land ungoverned in the gap: the
      // shadow has been live-mirroring every write since beginShadow()
      // above, and this generation is the floor a FUTURE open's watermark
      // verdict will trust once stamped.
      committedAtSwap = this.storage.committedGeneration?.() ?? committedAtStart!
    } catch (err) {
      oldManager.endShadow()
      prodLog.error(
        `[Brainy] repairIndex(): online metadata rebuild FAILED during the walk/fold — the ` +
          `live index is UNCHANGED (never swapped); reads keep serving the current ` +
          `(pre-rebuild) metadata index. Error: ${(err as Error).message}`
      )
      throw err
    }

    oldManager.endShadow()
    this.metadataIndex = shadow

    // NOW persist — the shadow is the SOLE owner of the shared storage keys
    // (nothing references `oldManager` any more; it never flushes again).
    shadow.stampWatermark(committedAtSwap)
    await shadow.flush()

    prodLog.warn(
      `[Brainy] repairIndex(): online metadata rebuild complete in ${Date.now() - startedAt}ms — ` +
        `swapped in a fresh index reflecting generation ${committedAtSwap}, zero read downtime.`
    )
  }

  /**
   * @description Rebuild indexes from persisted data if needed — THE OPEN-TIME
   * BUILD. Called once per open (init calls it; `repairIndex()`'s
   * write-quarantine lift calls it forced). Runs to completion BEFORE `init()`
   * returns: a needed rebuild is NEVER deferred to a read (there is no more
   * first-query lazy path — see {@link ensureIndexesLoaded}, which is a
   * read-time CHECK only). `disableAutoRebuild` no longer defers index
   * construction to the first query; see its JSDoc in `brainy.types.ts`.
   *
   * FIXES FOR CRITICAL BUGS:
   * - Bug #1: GraphAdjacencyIndex rebuild never called ✅ FIXED
   * - Bug #2: Early return blocks recovery when count=0 ✅ FIXED
   * - Bug #4: HNSW index has no rebuild mechanism ✅ FIXED
   * - Bug #5: disableAutoRebuild leaves indexes empty forever ✅ FIXED
   *
   * Production-grade rebuild with:
   * - Handles BILLIONS of entities via streaming pagination
   * - A provider's named {@link HealthReport} (when it exposes one) decides
   *   per-leg need; `isReady()` / a size heuristic decides otherwise — no
   *   dataset-size threshold gates whether the rebuild runs at open.
   * - Progress reporting for large datasets
   * - Parallel index rebuilds for performance
   * - Robust error recovery (continues on partial failures)
   *
   * @param force - Force the rebuild path to run even when no leg reports a need (used by `repairIndex()`'s write-quarantine lift).
   */
  private async rebuildIndexesIfNeeded(force = false): Promise<void> {
    try {
      // No instant fast-path here: the honest per-leg readiness checks below
      // are all O(1) (one bounded storage sample + each provider's health
      // report / size()/isReady()), and this method runs exactly once per
      // open. The removed shortcut keyed off `this.index.size() > 0`, a
      // dishonest proxy — it skipped the metadata and graph checks whenever
      // the vector happened to be warm, and it never fired on a real cold
      // process (the JS vector size is 0 until it loads).

      // BUG #2 FIX: Don't trust counts - check actual storage instead
      // Counts can be lost/corrupted in container restarts
      const entities = await this.storage.getNouns({ pagination: { limit: 1 } })
      const totalCount = entities.totalCount || 0

      // If storage is truly empty, no rebuild needed
      if (totalCount === 0 && entities.items.length === 0) {
        if (force && !this.config.silent) {
          console.log('✅ Storage empty - no rebuild needed')
        }
        // A fresh / empty brain's (empty) derived indexes are trivially current
        // for this build's epoch — stamp the version marker so the next open
        // recognises it as current and skips the drift rebuild.
        await this.stampBrainFormatIfNeeded()
        return
      }

      // Check if indexes need rebuilding
      const metadataStats = await this.metadataIndex.getStats()
      const hnswIndexSize = this.index.size()

      // Readiness contract: a provider's named {@link HealthReport} (when
      // exposed) is the authority — `serving === false` needs the rebuild,
      // full stop. Absent a health report, fall back to `isReady()` (an
      // mmap/disk-native index legitimately reports 0 resident entries while
      // fully durable on disk, so rebuilding it from canonical on every boot
      // would be the 48-seconds-per-restart class a production deployment
      // hit); absent BOTH, keep the per-leg empty-heuristic passed in.
      const legNeedsRebuild = (provider: unknown, emptyFallback: boolean): boolean => {
        const assessment = assessProviderHealth(provider)
        if (assessment.via === 'health-report') return assessment.readiness !== 'ready'
        if (assessment.via === 'is-ready') return assessment.readiness === 'not-ready'
        return emptyFallback
      }

      // Epoch-drift trigger: a format-version change makes EVERY derived index
      // suspect even when each is non-empty, so it forces a rebuild of all
      // three from the canonical records (not just the empty ones below).
      const epochStale = this._indexEpochStale

      // Migration LOCK (#18) deference: when a native provider holds the lock for
      // its one-time 7.x → 8.0 rebuild-from-canonical, brainy must NOT rebuild
      // that index — the provider owns it in place and clears the lock only after
      // it verifies and calls brain.stampBrainFormat(). Reads and writes are held
      // by awaitMigrationLock meanwhile (nothing serves from a half-built index).
      // Gated per-index, so a non-migrating sibling still rebuilds when it needs
      // to; a migrating provider is skipped even under epoch-drift or size()===0.
      // SELF-REBUILD DEFERENCE (the sibling of the migration lock, and the
      // reason a production open took 641 seconds): a provider that reports
      // `rebuildInProgress()` is ALREADY rebuilding its own index. Brainy must
      // neither start a second rebuild nor WAIT for the provider's — init()
      // returns, every other family serves, and that family's own doors refuse
      // by name (carrying this progress) until the provider reports serving.
      // A provider without the hook behaves exactly as before.
      const metadataRebuilding = assessProviderRebuild(this.metadataIndex)
      const vectorRebuilding = assessProviderRebuild(this.index)
      const graphRebuilding = assessProviderRebuild(this.graphIndex)
      for (const [leg, progress] of [
        ['metadata', metadataRebuilding],
        ['vector', vectorRebuilding],
        ['graph', graphRebuilding]
      ] as const) {
        if (progress) {
          prodLog.narrate(
            `[Brainy] open(): the ${leg} provider is ${describeRebuildProgress(progress)} — ` +
              `open does NOT wait for it. The brain opens now, every other family serves, and ` +
              `${leg} reads refuse by name until the provider reports itself serving.`
          )
        }
      }

      const metadataMigrating =
        this.providerIsMigrating(this.metadataIndex) || metadataRebuilding !== null
      const vectorMigrating = this.providerIsMigrating(this.index) || vectorRebuilding !== null
      const graphMigrating = this.providerIsMigrating(this.graphIndex) || graphRebuilding !== null
      // The epoch stamp certifies EVERY derived index, so it must not advance
      // while any family is still being built — by a migration lock or by the
      // provider itself.
      const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating

      // Per-leg decision, in precedence order: a migrating provider owns its
      // index (skip) → epoch drift forces a rebuild → the health-report/
      // isReady() authority decides → otherwise a per-leg fallback. The
      // fallbacks differ by leg because "empty" means different things:
      //   - METADATA: past the empty-store early-return, entities exist, so the
      //     id-mapper SHOULD have loaded entries — totalEntries===0 is a real
      //     load-failure signal, so rebuild (self-heal from canonical).
      //   - VECTOR: the JS vector has no passive cold-load — rebuild() IS its
      //     load path — so size()===0 correctly triggers the load.
      //   - GRAPH: entities do NOT imply edges, so size()===0 is a VALID empty
      //     state, not a load failure. The JS graph cold-loads (and self-heals
      //     against canonical) inside storage.getGraphIndex() BEFORE this gate,
      //     so it is already authoritative here; re-deriving would be spurious
      //     (a full O(E) verb scan on every open of an edgeless brain). It
      //     therefore rebuilds only on epoch drift or a native !isReady()/
      //     not-serving report. (verifyGraphAdjacencyLive is the query-time
      //     backstop — it refuses loudly, it never rebuilds.)
      const shouldRebuildMetadata =
        !metadataMigrating &&
        (epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0))

      // VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store
      // can hold canonical vectored nouns with NO derived vector index built.
      // `legNeedsRebuild`'s size-heuristic fallback (below) only fires off
      // `hnswIndexSize === 0`, and its health-report branch trusts a
      // provider's own `serving` verdict verbatim — but a provider's health
      // report can legitimately say `serving: true` while vector coverage is
      // honestly UNLEDGERED on ITS side too (an unledgered invariant never
      // flips serving), so neither signal alone can tell "genuinely empty"
      // apart from "never built". The canonical vectored-noun ledger
      // (`getCanonicalCounts().vectors.all` — Deliverable 1) is the
      // denominator that CAN tell them apart, and is compared here:
      //  - a CONFIDENT (non-suspect) ledger `> 0` while the reported node
      //    count is 0 is a proven coverage gap — force the build regardless
      //    of what a health report claims;
      //  - a CONFIDENT ledger `=== 0` while the node count is 0 proves there
      //    is nothing to load (e.g. every noun's embed is still deferred) —
      //    skip the size-heuristic fallback's blunt "always rebuild when
      //    empty" trigger, which otherwise wastes a full canonical walk for
      //    zero benefit on every cold open of such a store;
      //  - an unavailable/suspect ledger changes nothing — loud errors never
      //    quiet losses, so a doubtful ledger must never suppress a rebuild
      //    the old heuristic would have run.
      // The bare `isReady()` boolean (no report, no `unledgered` concept) is
      // NOT overridden — that signal is what fixed the 48-seconds-per-restart
      // regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a
      // disk-native provider legitimately reporting 0 resident while durable
      // on disk), and re-deriving it from a denominator the provider itself
      // has no way to consult would reopen exactly that regression.
      const vectorAssessment = assessProviderHealth(this.index)
      const vectorLedger = await this.storage.getCanonicalCounts?.()
      const vectorLedgerAll = vectorLedger?.vectors.all
      const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect
      const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0
      const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0

      let vectorNeedsRebuild: boolean
      if (vectorAssessment.via === 'is-ready') {
        // Bare isReady() stays authoritative and UNMODIFIED — see above.
        vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready'
      } else if (vectorAssessment.via === 'health-report') {
        vectorNeedsRebuild =
          vectorAssessment.readiness !== 'ready' ||
          (hnswIndexSize === 0 && vectorHasCoverageProof)
      } else {
        // size-heuristic / no provider (the built-in JS engine's own posture).
        vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty
      }

      const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild)

      // Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to
      // the defect this gate closes: a provider whose OWN health report
      // claims `serving: true` — an affirmative "I am ready" a caller would
      // otherwise trust outright — while the canonical ledger proves vector
      // coverage is missing. This is deliberately NARROWER than "any branch
      // where the ledger contributed to the decision":
      //  - the bare isReady() branch is untouched, as above (never in scope);
      //  - the health-report branch's OWN `readiness !== 'ready'` case is
      //    already an ordinary, PRE-EXISTING rebuild trigger (the provider
      //    admits not-ready) — not a ledger override, so not a "gap";
      //  - the size-heuristic/no-provider branch's rebuild-when-empty is the
      //    SAME blunt trigger the code always had (`hnswIndexSize === 0`)
      //    — the ledger only ever SUPPRESSES a rebuild there (the confirmed-
      //    empty case), it never forces one the old heuristic wouldn't
      //    already have run. Marking that branch a "gap" too made the
      //    FAIL-TYPED backstop fire on ordinary white-box tests that stub
      //    rebuild() as a no-op and pin `size()` at 0 to drive OTHER
      //    assertions (e.g. migration-deference's isMigrating() coverage) —
      //    those are not silent-empty defects, so they must open exactly as
      //    before (tests/unit/brainy/migration-deference.test.ts).
      const vectorCoverageGap =
        !vectorMigrating &&
        vectorAssessment.via === 'health-report' &&
        vectorAssessment.readiness === 'ready' &&
        hnswIndexSize === 0 &&
        vectorHasCoverageProof
      if (vectorCoverageGap) {
        prodLog.warn(
          `[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` +
            `ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` +
            `missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` +
            `silent-empty search results.`
        )
      }

      const shouldRebuildGraph =
        !graphMigrating &&
        (epochStale || legNeedsRebuild(this.graphIndex, false))

      const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph

      if (!needsRebuild && !force) {
        // All indexes report current — durably loaded (health-report/isReady/
        // size), or owned by a background migration. No rebuild needed.
        return
      }

      // Name exactly which legs rebuild — "all indexes" was a lie whenever
      // the durable legs were skipped (e.g. only the JS vector index loads
      // here on a warm reopen), and it misread as a whole-brain rebuild in
      // consumer boot logs.
      const rebuildingLegs = [
        shouldRebuildMetadata && 'metadata',
        shouldRebuildVector && 'vector',
        shouldRebuildGraph && 'graph'
      ]
        .filter(Boolean)
        .join(' + ')

      // ALWAYS narrated (prodLog, never the silent-suppressible console): there
      // is no more first-query lazy path — a rebuild that runs here BLOCKS
      // open() regardless of dataset size or `disableAutoRebuild`, so an
      // operator must see it in the boot log, not discover it as an
      // unexplained slow open.
      prodLog.warn(
        `[Brainy] open() is building/rebuilding the ${rebuildingLegs || 'no'} index(es) from ` +
          `${totalCount.toLocaleString()} stored entities — open blocks until the derived ` +
          `indexes serve; reads never build.`
      )

      // Before the graph rebuild, hydrate the entity id-mapper from the persisted
      // snapshot. A native int-keyed adjacency resolves every verb endpoint through
      // this mapper, so it must reflect the persisted int assignments BEFORE
      // graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints
      // and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s
      // ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()).
      if (shouldRebuildGraph) {
        await this.hydrateIdMapperForGraphRebuild()
      }

      // Rebuild the needed indexes in parallel for performance. Indexes load
      // their data from storage (no recomputation). An epoch drift (`epochStale`)
      // rebuilds every NON-migrating index regardless of its current size — the
      // on-disk format changed, so each is rebuilt from the canonical records. A
      // provider running its own background migration is skipped here (it owns
      // its index until it verifies-and-swaps).
      const rebuildStartTime = Date.now()
      // The vector leg's build door, by contract with the native provider: a
      // provider exposing fillFromCanonical() gets THAT call — idempotent, the
      // provider's own init runs it first so this is the backstop — never a
      // full rebuild() for a coverage gap. A PARTIAL shortfall deliberately
      // triggers nothing here: that is repair()'s operator door. The JS index
      // has no fill door and keeps its rebuild.
      const vectorBuild = (): Promise<unknown> => {
        const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise<unknown> })
          .fillFromCanonical
        if (vectorCoverageGap && typeof fillDoor === 'function') {
          prodLog.warn(
            `[Brainy] open(): vector coverage gap routes through the provider's ` +
              `fillFromCanonical() (idempotent canonical fill), not a full rebuild.`
          )
          return fillDoor.call(this.index)
        }
        return this.index.rebuild()
      }
      await Promise.all([
        shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
        shouldRebuildVector ? vectorBuild() : Promise.resolve(),
        shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
      ])

      const rebuildDuration = Date.now() - rebuildStartTime
      const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries
      const graphSizeAfter = await this.graphIndex.size()

      // Completion narration — ALWAYS via prodLog (see the pre-rebuild narration
      // above for why): the operator who saw "open() is building…" needs the
      // matching "…and it's done" line, with the numbers to confirm it worked.
      prodLog.warn(
        `[Brainy] open() finished building derived indexes in ${rebuildDuration}ms: ` +
          `metadata=${metadataCountAfter} entries, vector=${this.index.size()} nodes, ` +
          `graph=${graphSizeAfter} relationships.`
      )

      if (!this.config.silent) {
        console.log(
          `All indexes rebuilt in ${rebuildDuration}ms:\n` +
          `   - Metadata: ${metadataCountAfter} entries\n` +
          `   - HNSW Vector: ${this.index.size()} nodes\n` +
          `   - Graph Adjacency: ${graphSizeAfter} relationships`
        )
      }

      // Consistency verification: metadata index must match storage entity count.
      // If mismatch, the rebuild missed entities — force a second attempt. Skipped
      // when the metadata provider holds the migration lock: a 0 count there
      // reflects its in-place rebuild in progress, not a missed rebuild, so
      // forcing a second rebuild would collide with the provider's own.
      // THREE states, not two. `metadataMigrating` above is true for a
      // provider holding the migration lock AND for one that reports it is
      // rebuilding itself — a provider whose rebuild() returns once the
      // rebuild is OWNED AND RUNNING (online, its doors refusing by name)
      // legitimately reports 0 entries here, and calling that CRITICAL would
      // print a false alarm and kick a redundant second rebuild on every
      // first contact. The check's real class — a rebuild that ran to
      // completion and produced nothing — is untouched: a provider reporting
      // 0 entries with NO rebuild in progress still trips it.
      if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) {
        console.error(
          `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` +
          `Forcing second rebuild.`
        )
        await this.metadataIndex.rebuild()
        const secondAttempt = (await this.metadataIndex.getStats()).totalEntries
        console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
      }

      // Vector coverage verification: the coverage-gap rebuild above (see
      // `vectorCoverageGap`) MUST have actually restored the ledger's
      // vectored nouns. A provider that STILL reports 0 nodes after its own
      // rebuild() ran — no JS (or provider) fallback could build from what's
      // on disk — cannot silently complete open(): search would then serve
      // empty results with no signal, exactly the defect this gate closes.
      // FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a
      // successful open.
      if (vectorCoverageGap && this.index.size() === 0) {
        throw new VectorIndexNotReadyError(
          `open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` +
            `index still reports 0 node(s) after rebuild() — the derived vector index could not ` +
            `be restored from canonical. Refusing to serve silent-empty search results; ` +
            `investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` +
            `after restoring the underlying data.`
        )
      }

      // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
      // have now rebuilt and verified, so they match this build's epoch —
      // re-stamp the marker LAST, only here. A crash anywhere above leaves the
      // old / absent marker, so the next open re-detects the drift and re-runs
      // the (idempotent) rebuild; the marker is never advanced ahead of the
      // indexes it certifies. A no-op when the epoch was already current.
      //
      // EXCEPT while a provider holds the migration lock: brainy deferred that
      // index's rebuild, so it is NOT yet at this epoch. The marker (which
      // certifies ALL derived indexes) must not advance ahead of the index the
      // provider is still rebuilding in place — the provider calls
      // brain.stampBrainFormat() itself once it has verified parity.
      if (!anyMigrating) {
        await this.stampBrainFormatIfNeeded()
      }

    } catch (error) {
      // A storage READ failure here is surfaced by getNouns/getVerbs as a named
      // BrainyError — it means we could not even read the store to decide whether
      // a rebuild is needed (or to perform it). That is a hard init failure, not a
      // "start anyway" condition: booting a brain whose data we couldn't read would
      // serve empty/incomplete results with no signal (the silent-failure class the
      // 8.0 contract forbids). Re-throw it loud.
      if (error instanceof BrainyError) throw error
      // Any other rebuild hiccup is non-fatal — the brain may start on partially
      // rebuilt indexes — but it is recorded as a queryable degraded state
      // (surfaced via checkHealth) and escalated to error-level logging, rather
      // than only emitting a console.warn that is trivially lost.
      this._indexRebuildFailed = error instanceof Error ? error : new Error(String(error))
      console.error('[Brainy] Index rebuild failed; starting in a DEGRADED state (queries may be incomplete):', error)
    }
  }

  /**
   * @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker
   * (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} /
   * {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent
   * or carried a drifted epoch ({@link _indexEpochStale}). Called at the
   * verified-completion points of {@link rebuildIndexesIfNeeded} (after the
   * derived-index rebuild, or on the empty-storage fast path), so the marker is
   * never advanced ahead of the indexes it certifies — a crash before this
   * point re-triggers the idempotent rebuild on the next open. Readers never
   * write, so this is a no-op in reader mode.
   */
  private async stampBrainFormatIfNeeded(): Promise<void> {
    if (this.isReadOnly) return
    if (!this._indexEpochStale) return
    await writeBrainFormat(this.storage)
    this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
    this._indexEpochStale = false
    // The JS inline rebuild verified + stamped — drop the pre-upgrade backup.
    await this.removeMigrationBackupSafe()
  }

  /**
   * @description Whether an index provider holds the coordinated migration LOCK
   * (#18). A native provider's `init()` flips an OPTIONAL `isMigrating(): boolean`
   * to `true` the moment it detects a large 7.x epoch-drift and keeps it `true`
   * while it rebuilds all derived indexes IN PLACE from the canonical records,
   * clearing it only after it has verified parity and stamped the marker. While
   * `true`, brainy BLOCKS/QUEUES data-plane reads AND writes on the lock (see
   * {@link awaitMigrationLock}) so no operation touches a half-built index —
   * the "unknown/halfway state" is closed by waiting for a known-good one.
   * Feature-detected: a provider that omits `isMigrating` (every JS index) is
   * treated as not migrating, so behavior is unchanged.
   * @param provider - A registered index provider (metadata / vector / graph).
   * @returns `true` only when the provider exposes `isMigrating()` and it reports
   *   an in-flight migration.
   */
  private providerIsMigrating(provider: unknown): boolean {
    return (
      provider != null &&
      typeof (provider as { isMigrating?: () => boolean }).isMigrating === 'function' &&
      (provider as { isMigrating: () => boolean }).isMigrating() === true
    )
  }

  /**
   * @description `true` when ANY index provider (metadata / vector / graph) holds
   * the migration LOCK. The single predicate the data-plane gate and the status
   * surface consult; a brain with no migrating provider evaluates three cheap
   * feature-detects and returns `false`.
   */
  private anyProviderMigrating(): boolean {
    return (
      this.providerIsMigrating(this.metadataIndex) ||
      this.providerIsMigrating(this.index) ||
      this.providerIsMigrating(this.graphIndex)
    )
  }

  /**
   * @description The provider instance backing a given index {@link IndexFamily}.
   * The single place the family label maps to the concrete provider, so the
   * scoped migration gate and any future family-aware routing agree.
   */
  private providerForFamily(family: IndexFamily): unknown {
    switch (family) {
      case 'vector':
        return this.index
      case 'metadata':
        return this.metadataIndex
      case 'graph':
        return this.graphIndex
    }
  }

  /**
   * @description Whether the migration LOCK should hold for an operation that
   * depends on the given index families:
   * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the
   *   conservative default for writes and unclassified reads — a write touches
   *   every index).
   * - `needs === []` → NEVER: a canonical-storage read consults no derived index,
   *   so a migration elsewhere is irrelevant; it serves immediately.
   * - `needs = [families]` → SCOPED: only those families' providers count, so a
   *   read served from a healthy family is not blocked by an unrelated family's
   *   one-time migration.
   */
  private neededFamiliesMigrating(needs?: IndexFamily[]): boolean {
    if (needs === undefined) return this.anyProviderMigrating()
    for (const family of needs) {
      if (this.providerIsMigrating(this.providerForFamily(family))) return true
    }
    return false
  }

  /**
   * @description The index families a {@link find} query consults, derived from
   * its params — the read-side input to the family-scoped migration gate. A
   * vector / near / semantic query needs `vector`; a `where` / type / subtype /
   * service filter or an aggregate needs `metadata`; a `connected` traversal
   * needs `graph`. A query combining shapes needs the union. Mirrors find()'s
   * own criteria split so the gate and the executor never disagree.
   */
  private queryIndexFamilies(params: FindParams<T>): IndexFamily[] {
    const needs: IndexFamily[] = []
    if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector')
    if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata')
    if (params.connected) needs.push('graph')
    return needs
  }

  /**
   * @description Rich migration progress relayed verbatim from whichever provider
   * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`.
   * The provider owns the rebuild, so it owns the truth of the percentage; brainy
   * surfaces it through {@link getIndexStatus} and {@link health}. Returns `null`
   * when no provider reports structured progress (brainy then shows only
   * `migrating: true` + `elapsedMs`). Feature-detected and best-effort.
   */
  private providerMigrationStatus(): MigrationProgress | null {
    for (const provider of [this.metadataIndex, this.index, this.graphIndex]) {
      const fn = (provider as { migrationStatus?: () => unknown }).migrationStatus
      if (typeof fn === 'function') {
        try {
          const status = fn.call(provider)
          if (status && typeof status === 'object') {
            return status as MigrationProgress
          }
        } catch {
          // best-effort: a provider status hiccup never breaks the gate/probe
        }
      }
    }
    return null
  }

  /**
   * @description Block-and-queue on the coordinated migration LOCK (#18). Called
   * at the {@link ensureInitialized} choke point (and once inside `init()` before
   * VFS bootstrap), so a data-plane operation waits here while a native provider
   * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the
   * correct answer, never a partial read of a half-built index and never a lost
   * write. A brain that is not migrating pays a single boolean check and returns
   * immediately.
   *
   * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the
   * wait holds ONLY for a migration of one of those families. A read served
   * entirely from canonical storage (`needs: []`) or from a healthy family never
   * blocks on an unrelated family's migration. `needs` omitted = the conservative
   * whole-brain wait (writes touch every index; startup wants all of them ready).
   * @param needs - Index families this operation consults, or `undefined` for the
   *   whole-brain wait.
   *
   * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The
   * migration itself is unbounded — the native provider rebuilds a billion-scale
   * brain for as long as it needs; brainy cannot and does not abort it. The lock
   * is polled every {@link Brainy.MIGRATION_POLL_INTERVAL_MS}; a small/medium
   * upgrade clears well within `migrationWaitTimeoutMs` (default 30 s) and the
   * caller resumes transparently. If the wait outlives the budget, a retryable
   * {@link MigrationInProgressError} is thrown — the upgrade keeps running in the
   * background, so the caller retries, watches `getIndexStatus().migration`
   * (never gated — the primary large-upgrade signal for a readiness probe), or,
   * for a very large brain, runs the offline migrator. The block/release lines
   * log once per window; the flags reset on release so a later upgrade re-logs.
   */
  private async awaitMigrationLock(needs?: IndexFamily[]): Promise<void> {
    if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case

    const startedAt = Date.now()
    const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000

    if (!this._migrationBlockLogged) {
      this._migrationBlockLogged = true
      prodLog.info(
        '[Brainy] Auto-upgrade in progress (7.x → 8.0): reads and writes are blocked ' +
          'until the rebuild completes. This is automatic and one-time; data is safe.'
      )
    }

    while (this.neededFamiliesMigrating(needs)) {
      const remaining = timeoutMs - (Date.now() - startedAt)
      if (remaining <= 0) {
        const status = this.providerMigrationStatus()
        throw new MigrationInProgressError(
          `Brain is still upgrading (7.x → 8.0)` +
            (status?.percent !== undefined ? `, ${Math.round(status.percent)}% complete` : '') +
            ` after ${Math.round((Date.now() - startedAt) / 1000)}s. The wait timed out; the upgrade ` +
            `continues in the background. Retry shortly, watch brain.getIndexStatus().migration for ` +
            `progress, raise config.migrationWaitTimeoutMs to wait longer, or for a very large brain ` +
            `run the offline migrator. Nothing is lost.`,
          Date.now() - startedAt,
          status?.percent
        )
      }
      await new Promise((r) => setTimeout(r, Math.min(Brainy.MIGRATION_POLL_INTERVAL_MS, remaining)))
    }

    if (this._migrationBlockLogged && !this._migrationReleaseLogged) {
      this._migrationReleaseLogged = true
      prodLog.info(
        `[Brainy] Auto-upgrade complete in ${Math.round((Date.now() - startedAt) / 1000)}s — ` +
          'reads and writes resumed.'
      )
    }
    // Reset the once-per-window guards so a subsequent upgrade in this same
    // instance re-logs cleanly and re-clocks its elapsed (a fresh observed-at).
    this._migrationBlockLogged = false
    this._migrationReleaseLogged = false
    this._migrationObservedAt = null
  }

  /**
   * @description The lock-exempt observability snapshot consumed by
   * {@link getIndexStatus} and {@link health}: whether the brain is upgrading and,
   * if so, the provider's structured {@link MigrationProgress} enriched with a
   * `startedAt` / `elapsedMs` that brainy tracks itself (so a provider that omits
   * timing still yields a live elapsed). Reads the migration flag without waiting,
   * so a readiness probe can always report `migrating` → HTTP 503 + Retry-After
   * rather than routing traffic into a half-built index. Lazily stamps the
   * observed-at clock on first sight and clears it once the lock releases.
   */
  private migrationSnapshot(): { migrating: boolean; migration?: MigrationProgress } {
    if (!this.anyProviderMigrating()) {
      this._migrationObservedAt = null
      return { migrating: false }
    }
    if (this._migrationObservedAt === null) {
      this._migrationObservedAt = Date.now()
    }
    const provided = this.providerMigrationStatus()
    return {
      migrating: true,
      migration: {
        ...provided,
        startedAt: provided?.startedAt ?? this._migrationObservedAt,
        elapsedMs: provided?.elapsedMs ?? Date.now() - this._migrationObservedAt
      }
    }
  }

  /**
   * @description Take the pre-upgrade backup (default-on) before a native provider
   * rebuilds anything for a 7.x → 8.0 migration. Called at open, once the on-disk
   * format is known stale, BEFORE the providers init. Feature-detected + best-effort:
   * a backend without `createMigrationBackup` (memory) or a store with no data is a
   * no-op, and a backup failure NEVER blocks the upgrade (the migration is itself
   * safe — it only reads canonical, and self-heals on re-open). The snapshot is
   * removed once the upgrade verifies ({@link removeMigrationBackupSafe}) and
   * retained on failure for rollback.
   */
  private async createMigrationBackupIfNeeded(): Promise<void> {
    const storage = this.storage as StorageAdapter & {
      createMigrationBackup?: () => Promise<string | null>
    }
    if (typeof storage.createMigrationBackup !== 'function') return
    try {
      const backupPath = await storage.createMigrationBackup()
      if (backupPath) {
        this._migrationBackupPath = backupPath
        prodLog.info(
          `[Brainy] Pre-upgrade backup created at ${backupPath} (hard-link snapshot; ~zero ` +
            `cost/space). Removed automatically once the 7.x→8.0 upgrade verifies; retained ` +
            `for rollback if it fails. Opt out with { migrationBackup: false }.`
        )
      }
    } catch (error) {
      prodLog.warn(
        `[Brainy] Pre-upgrade backup could not be created (continuing — the upgrade is safe ` +
          `and self-heals): ${error instanceof Error ? error.message : String(error)}`
      )
    }
  }

  /**
   * @description Remove the pre-upgrade backup once the migration has verified and
   * stamped the format marker. Called from both stamp paths (the JS inline rebuild
   * and the native provider's `stampBrainFormat()`), gated on a backup having been
   * taken this open. Best-effort: a removal failure leaves a harmless backup dir.
   */
  private async removeMigrationBackupSafe(): Promise<void> {
    if (!this._migrationBackupPath) return
    // Backup-parity gate: the index rebuild "success" that calls this does NOT
    // certify the VFS. If the on-open blob adoption could not fully bridge the
    // 7.x `_cow/` blobs, the upgrade is not verifiably complete for VFS content
    // — retain the backup so the operator can recover from it, and log why.
    if (this._vfsBlobAdoptionIncomplete) {
      if (!this.config.silent) {
        console.warn(
          `[brainy] Retaining the pre-upgrade backup at ${this._migrationBackupPath}: ` +
            `some VFS content blobs could not be adopted from _cow/ (see the earlier warning). ` +
            `Resolve those before discarding the backup.`
        )
      }
      return
    }
    const backupPath = this._migrationBackupPath
    this._migrationBackupPath = null
    const storage = this.storage as StorageAdapter & {
      removeMigrationBackup?: (location: string) => Promise<void>
    }
    if (typeof storage.removeMigrationBackup !== 'function') return
    try {
      await storage.removeMigrationBackup(backupPath)
      prodLog.info(`[Brainy] Upgrade verified — pre-upgrade backup at ${backupPath} removed.`)
    } catch {
      // best-effort — a leftover backup dir is harmless
    }
  }

  /**
   * Check health of metadata indexes
   *
   * Returns validation result indicating whether indexes are healthy
   * or corrupted (e.g., from the update() field asymmetry bug).
   *
   * This check was previously run on every init(), causing significant
   * overhead on cloud storage (90+ sequential reads for 30-field datasets).
   * Now available as an on-demand diagnostic method.
   */
  async checkHealth(): Promise<{
    healthy: boolean
    avgEntriesPerEntity: number
    entityCount: number
    indexEntryCount: number
    recommendation: string | null
  }> {
    // Lock-exempt: diagnostics must answer while the brain upgrades.
    await this.ensureInitialized({ bypassMigrationLock: true })
    const result = await this.metadataIndex.validateConsistency()
    // Fold in a non-fatal index-rebuild failure recorded at init() so the degraded
    // state is observable through the same health surface (not just the console).
    if (this._indexRebuildFailed) {
      return {
        ...result,
        healthy: false,
        recommendation:
          `Index rebuild failed at init() (degraded — queries may be incomplete): ` +
          `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` +
          (result.recommendation ? ` Also: ${result.recommendation}` : '')
      }
    }
    if (this._indexDegradedIds.size > 0) {
      return {
        ...result,
        healthy: false,
        recommendation:
          `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
          `failed-rollback recovery — the derived index may be incomplete for them. ` +
          `Run repairIndex() to reconcile.` +
          (result.recommendation ? ` Also: ${result.recommendation}` : '')
      }
    }
    return result
  }

  /**
   * Detect and repair corrupted metadata indexes.
   *
   * Runs corruption detection and auto-rebuilds if corruption is found. This is
   * the equivalent of the old init()-time corruption check, now available as an
   * explicit operation.
   *
   * It is ALSO the recovery path for a write-quarantine: when a transaction's
   * rollback fails and leaves the store inconsistent ({@link StoreInconsistentError}),
   * writes are refused until this method reconciles the derived indexes against
   * canonical storage (a forced rebuild — orphaned/lost records reflected
   * consistently) and lifts the quarantine. Canonical is the source of truth: a
   * genuinely lost record cannot be resurrected here (restore from a snapshot for
   * that), but the store is made internally consistent and writes re-enabled.
   */
  /**
   * Emit ONE loud warning per degraded window when a read is served while the
   * derived index is known-incomplete — either a non-fatal init rebuild failure
   * ({@link _indexRebuildFailed}) or an adopt-forward degraded commit
   * ({@link _indexDegradedIds}). Reads still return (canonical is the source of
   * truth), but the caller is told the result may be partial and how to heal it.
   * No-op when healthy or when the brain is configured `silent`.
   * @param op - The read method name for the message (e.g. `'find'`, `'get'`).
   */
  private warnIfReadsDegraded(op: string): void {
    const degraded =
      this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0
    if (!degraded) {
      this._degradedReadWarned = false
      return
    }
    if (this._degradedReadWarned || this.config.silent) return
    this._degradedReadWarned = true
    const reason = this._indexRebuildFailed
      ? `index rebuild failed at init() (${this._indexRebuildFailed.message})`
      : `${this._indexDegradedIds.size} record(s) committed in a degraded state`
    prodLog.warn(
      `[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` +
      `(${reason}). Results may be missing entries — run repairIndex() to ` +
      `reconcile the derived indexes against canonical storage.`
    )
  }

  /**
   * Read-only graph-truth audit — proves (or disproves) that relationship
   * reads return canonical truth on THIS brain, and classifies every
   * discrepancy into its failure family:
   *
   * - `missingFromReads` — canonical verb records the read path omits
   *   (PRESENT BUT INVISIBLE: adjacency/membership staleness)
   * - `danglingEndpoints` — verbs whose endpoint entity is gone (SCAR class)
   * - `readOnlyVerbIds` — read-path edges with no canonical record (GHOSTS)
   *
   * Design-hidden edges (internal/system visibility) are counted separately —
   * the audit reads with every visibility tier included, so intentional
   * hiding is never misclassified as index loss. Mutates nothing; safe on a
   * live brain (cost: one canonical walk + one indexed read per source).
   * Run it after any engine upgrade, restore, or migration; a `coherent`
   * report is the verified statement that `related()`/`readdir` can be
   * trusted. If it reports discrepancies, `repairIndex()` is the sanctioned
   * heal — re-run the audit afterwards to prove the repair.
   *
   * @param options.maxExamples - Cap per example list (counts stay exact). Default 100.
   * @returns The full audit report; also narrated via logs (loud on incoherence).
   * @since 8.6.0
   */
  async auditGraph(options: { maxExamples?: number } = {}): Promise<GraphAuditReport> {
    await this.ensureInitialized({ needs: ['graph'] })

    const PAGE = 1000
    return runGraphAudit(
      {
        eachNounId: async (consume) => {
          let cursor: string | undefined
          let offset = 0
          for (;;) {
            const page = await (this.storage as unknown as {
              getNounIdsWithPagination(o: {
                limit: number
                offset?: number
                cursor?: string
              }): Promise<{ ids: string[]; hasMore: boolean; nextCursor?: string }>
            }).getNounIdsWithPagination(
              cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
            )
            for (const id of page.ids) consume(id)
            if (!page.hasMore || page.ids.length === 0) break
            if (page.nextCursor) cursor = page.nextCursor
            else offset += page.ids.length
          }
        },
        eachVerb: async (consume) => {
          let cursor: string | undefined
          let offset = 0
          for (;;) {
            const page = await this.storage.getVerbs({
              pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
            })
            for (const verb of page.items) {
              const v = verb as unknown as Record<string, unknown>
              consume({
                id: String(v.id),
                type: String(v.verb ?? 'unknown'),
                sourceId: String(v.sourceId),
                targetId: String(v.targetId),
                visibility: typeof v.visibility === 'string' ? v.visibility : undefined
              })
            }
            if (!page.hasMore || page.items.length === 0) break
            if (page.nextCursor) cursor = page.nextCursor
            else offset += page.items.length
          }
        },
        readRelationsFrom: async (sourceId) =>
          this.related({
            from: sourceId,
            includeInternal: true,
            includeSystem: true,
            limit: 100000
          })
      },
      options
    )
  }

  /**
   * @description The ceremony door for index repair. Bare `repairIndex()` is
   * REPORT-DRIVEN, exactly as before: it prunes orphans, recomputes count
   * rollups, reconciles VFS containment, and — for the three derived-index
   * providers — consults each one's `validateInvariants()` and rebuilds only
   * a family whose failing invariant asks for it (`heal: 'rebuild'`).
   *
   * `options.rebuild` is the EXPLICIT operator override: name one or more
   * families (or `'all'`) to rebuild them UNCONDITIONALLY — no invariant is
   * consulted, JS or native provider alike. Use it when an operator has
   * independent reason to believe a family needs reconciling regardless of
   * what its own self-report says (a report can only be as honest as the
   * provider that produced it). A family named here is recorded as its own
   * `provider:<name>` row with `rebuilt: true` and
   * `reason: 'explicit rebuild requested'`, and is SKIPPED by the normal
   * invariant-driven pass (it was already rebuilt unconditionally — a second,
   * report-driven pass over the same family would be redundant at best).
   *
   * NARRATION IS PART OF THE CONTRACT. A repair on a production store ran for
   * more than thirty minutes at a full core with NOT ONE log line between its
   * start and its end while the doors kept serving; the operator could tell it
   * was alive only from `top`. Every phase now announces itself before it
   * works, a heartbeat names the phase still running every five seconds, and
   * each phase reports its own wall — carried in the receipt as
   * `durationMs` per family, so nobody has to infer progress from CPU.
   *
   * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`).
   * @returns The full per-family receipt (see {@link RepairReport}); also narrated as it goes.
   */
  async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise<RepairReport> {
    await this.ensureInitialized()
    // A repair recounts, prunes and rebuilds outside the commit paths; the
    // dirty witness is set so a caller's flush after a repair does its normal
    // work rather than finding the brain "clean".
    this._dirtySinceLastFlush = true
    const startedAt = Date.now()
    const families: RepairFamilyReport[] = []

    // THE REPAIR HEARTBEAT — the same law the open obeys: no stretch of work
    // may be silent for more than REPAIR_HEARTBEAT_MS. Unref'd (it never holds
    // a process open) and cleared in the `finally` below.
    const REPAIR_HEARTBEAT_MS = 5_000
    let currentPhase = 'starting'
    let currentPhaseCause = 'preparing the repair'
    let phaseStartedAt = Date.now()
    const heartbeat = setInterval(() => {
      prodLog.narrate(
        `[Brainy] repairIndex: still in "${currentPhase}" after ` +
          `${Math.round((Date.now() - phaseStartedAt) / 1000)}s ` +
          `(${Math.round((Date.now() - startedAt) / 1000)}s into the repair) — ${currentPhaseCause}`
      )
    }, REPAIR_HEARTBEAT_MS)
    if (typeof heartbeat.unref === 'function') heartbeat.unref()

    /** Announce a phase before it does any work, and start its clock. */
    const beginPhase = (name: string, cause: string): void => {
      currentPhase = name
      currentPhaseCause = cause
      phaseStartedAt = Date.now()
      prodLog.narrate(`[Brainy] repairIndex: "${name}" started — ${cause}`)
    }

    /**
     * Close the current phase: stamp its wall into the receipt row and say
     * what it did. Every family row carries its own `durationMs`.
     */
    const record = (family: string, entry: Omit<RepairFamilyReport, 'family' | 'durationMs'>): void => {
      const durationMs = Date.now() - phaseStartedAt
      families.push({ family, ...entry, durationMs })
      prodLog.narrate(
        `[Brainy] repairIndex: "${family}" finished in ${durationMs}ms — ` +
          (entry.checked
            ? `${entry.healed} heal(s)${entry.rebuilt ? ', rebuilt' : ''}` +
              (entry.detail ? ` (${entry.detail})` : '')
            : `skipped (${entry.skipped ?? entry.reason ?? 'no reason given'})`)
      )
      phaseStartedAt = Date.now()
    }

    try {
      return await this.runRepairIndexPhases(options, families, record, beginPhase, startedAt)
    } finally {
      clearInterval(heartbeat)
    }
  }

  /**
   * @description The phases of {@link repairIndex}, separated so its heartbeat
   * can live in a `finally` around them. Not a public door — see `repairIndex`
   * for the contract.
   * @param options - As `repairIndex`.
   * @param families - The receipt rows being accumulated.
   * @param record - Closes a phase: stamps its wall and narrates its outcome.
   * @param beginPhase - Announces a phase before it works.
   * @param startedAt - When the repair began, for the closing line.
   * @returns The full receipt.
   */
  private async runRepairIndexPhases(
    options: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' } | undefined,
    families: RepairFamilyReport[],
    record: (family: string, entry: Omit<RepairFamilyReport, 'family' | 'durationMs'>) => void,
    beginPhase: (name: string, cause: string) => void,
    startedAt: number
  ): Promise<RepairReport> {

    // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete
    // defect: a delete that removed the metadata (content) leg but left the
    // vector leg + the entity directory (a "ghost"), or left an empty directory
    // (a "scar"). These are not live entities (getNoun needs the content leg)
    // yet inflate enumerated counts and confuse locator resolution. Feature-
    // detected (filesystem-only — key/prefix stores have no orphan containers);
    // recompute counts afterward so the totals stop counting the ghosts.
    const pruner = this.storage as {
      pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }>
      rebuildTypeCounts?: () => Promise<void>
      rebuildSubtypeCounts?: () => Promise<void>
    }
    if (typeof pruner.pruneOrphanedEntities === 'function') {
      beginPhase(
        'orphaned-containers',
        'walking every canonical id directory for ghost/scar containers left by a partial delete'
      )
      const orphans = await pruner.pruneOrphanedEntities()
      const pruned = orphans.nouns.length + orphans.verbs.length
      record('orphaned-containers', {
        checked: true,
        healed: pruned,
        ...(pruned > 0
          ? { detail: `${orphans.nouns.length} noun + ${orphans.verbs.length} verb container(s) pruned` }
          : {})
      })
      if (pruned > 0) {
        prodLog.narrate(
          `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
            `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
            `partial delete.`
        )
      }
    } else {
      record('orphaned-containers', { checked: false, healed: 0, skipped: 'storage has no container model' })
    }
    // SANCTIONED RECOUNT — unconditional, not gated on orphans found: the
    // persisted counters can be inflated over perfectly clean shelves (deletes
    // whose decrement was skipped by the removed-record re-read), and
    // Math.max(totalNounCount, scanned) means an inflated scalar can never
    // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup
    // (scalar totals + per-type maps + type-statistics arrays) from one
    // canonical walk and persists them.
    beginPhase(
      'count-rollups',
      'ONE canonical walk recomputing every counter rollup — scalar totals, per-type maps, type statistics'
    )
    await pruner.rebuildTypeCounts?.()
    await pruner.rebuildSubtypeCounts?.()
    record('count-rollups', {
      checked: typeof pruner.rebuildTypeCounts === 'function',
      healed: 0,
      detail: typeof pruner.rebuildTypeCounts === 'function'
        ? 'recomputed from one canonical walk (unconditional)'
        : undefined,
      ...(typeof pruner.rebuildTypeCounts !== 'function' ? { skipped: 'storage has no count rollups' } : {})
    })

    // The recount changed the rollup truth — re-stamp the entity tree so the
    // stamp's invariants match the healed counters (repair leaves a coherent
    // stamp, not a stale one that warns on the next open).
    await this.stampEntityTree()

    // VFS containment reconciliation: heal "cosmetic ghost" edges left by
    // pre-fix renames (an entity Contains-linked from BOTH its old and new
    // directory — readdir listed it in two places) and duplicate edges from
    // concurrent writers. Canonical metadata.path is the truth; only VFS
    // containment edges are touched. Loud per repair.
    if (this._vfsInitialized && this._vfs) {
      beginPhase(
        'vfs-containment',
        'reconciling VFS containment edges against canonical metadata.path'
      )
      const containment = await this._vfs.repairContainment()
      record('vfs-containment', {
        checked: true,
        healed: containment.removed + containment.restored,
        ...(containment.removed + containment.restored > 0
          ? { detail: `${containment.removed} stale edge(s) removed, ${containment.restored} restored` }
          : {})
      })
      if (containment.removed + containment.restored > 0) {
        prodLog.narrate(
          `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` +
            `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).`
        )
      }
    }

    if (!this._vfsInitialized || !this._vfs) {
      record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' })
    }

    beginPhase(
      'metadata-corruption',
      'detect-and-repair pass over the metadata index'
    )
    await this.metadataIndex.detectAndRepairCorruption()
    record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' })
    // Lift a failed-rollback write-quarantine: force a full rebuild so the
    // derived indexes are provably reconciled with canonical, then clear the
    // flag so writes resume.
    if (this.storeInconsistency) {
      beginPhase(
        'write-quarantine',
        'full derived-index rebuild to lift the quarantine set by a failed transaction rollback'
      )
      await this.rebuildIndexesIfNeeded(true)
      const cleared = this.storeInconsistency
      record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` })
      this.storeInconsistency = null
      prodLog.narrate(
        `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` +
        `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` +
        `Writes are re-enabled.`
      )
    }
    // THE CEREMONY DOOR: an explicit `options.rebuild` names a family (or
    // 'all') to rebuild UNCONDITIONALLY — no invariant consulted. Resolved
    // here so the loop below can skip a family's normal report-driven pass
    // once its unconditional rebuild has already run.
    const explicitRebuildFamilies: ReadonlySet<'metadata' | 'vector' | 'graph'> =
      options?.rebuild === 'all'
        ? new Set<'metadata' | 'vector' | 'graph'>(['metadata', 'vector', 'graph'])
        : new Set(options?.rebuild ?? [])

    // Cross-layer repair: repairIndex must reconcile NATIVE derived
    // state from canonical, not just the JS metadata index. Consult each provider's
    // own validateInvariants() and rebuild any whose failing invariant asks for it
    // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption().
    const providerFamilies: ReadonlyArray<readonly ['metadata' | 'vector' | 'graph', unknown]> = [
      ['metadata', this.metadataIndex],
      ['vector', this.index],
      ['graph', this.graphIndex]
    ]
    for (const [familyName, provider] of providerFamilies) {
      if (explicitRebuildFamilies.has(familyName)) {
        const p = provider as { rebuild?: () => Promise<void> } | null
        if (!p || typeof p.rebuild !== 'function') {
          record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' })
          continue
        }
        beginPhase(
          `provider:${familyName}`,
          `explicit rebuild requested — rebuilding '${familyName}' unconditionally, no invariant consulted`
        )
        // The metadata family routes through the online build-beside
        // orchestrator (B3 D3) instead of the provider's own rebuild() —
        // zero read downtime when a fact log is available, narrated
        // fallback to the blocking rebuild() otherwise.
        if (familyName === 'metadata') {
          await this.rebuildMetadataIndexOnline()
        } else {
          await p.rebuild()
        }
        record(`provider:${familyName}`, {
          checked: true,
          healed: 1,
          rebuilt: true,
          reason: 'explicit rebuild requested'
        })
        prodLog.narrate(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`)
        continue
      }

      const p = provider as {
        validateInvariants?: () => Promise<ProviderInvariantReport>
        rebuild?: () => Promise<void>
      } | null
      if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') {
        beginPhase(`provider:${familyName}`, 'checking the provider contract')
        record(`provider:${familyName}`, {
          checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract'
        })
        continue
      }
      beginPhase(
        `provider:${familyName}`,
        `reading the '${familyName}' provider's own invariant report, then healing only what it asks for`
      )
      let report: ProviderInvariantReport
      try {
        report = await p.validateInvariants()
      } catch (err) {
        record(`provider:${familyName}`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` })
        continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here
      }
      if (report.healthy) {
        record(`provider:${report.provider}`, { checked: true, healed: 0 })
        continue
      }
      if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) {
        record(`provider:${report.provider}`, {
          checked: true, healed: 1,
          detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})`
        })
        prodLog.narrate(
          `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` +
            `requiring a rebuild — reconciling its derived state from canonical.`
        )
        // See the explicit-rebuild branch above: 'metadata' routes through
        // the online build-beside orchestrator (B3 D3).
        if (familyName === 'metadata') {
          await this.rebuildMetadataIndexOnline()
        } else {
          await p.rebuild()
        }
      } else if (
        report.invariants.some((i) => !i.holds && i.heal === 'repair') &&
        typeof (provider as { repair?: () => Promise<unknown> }).repair === 'function'
      ) {
        // INCREMENTAL HEAL ROUTING (ADR-008 D4): a failing verdict whose heal
        // is 'repair' routes to the provider's own repair() — O(missing),
        // re-posting exactly what its ledger names, never a store-sized
        // rebuild. The return shape is the provider's own; the RE-READ of the
        // report is what decides success (the acceptance meta-pin's law: run
        // the named heal once, re-read, nothing may still fail the same way).
        const failingRepairs = report.invariants
          .filter((i) => !i.holds && i.heal === 'repair')
          .map((i) => i.name)
        prodLog.narrate(
          `[Brainy] repairIndex(): provider '${report.provider}' asks for an incremental ` +
            `repair (${failingRepairs.join(', ')}) — running its own repair().`
        )
        await (provider as { repair: () => Promise<unknown> }).repair()
        let cleared = false
        let after: ProviderInvariantReport | null = null
        try {
          after = await p.validateInvariants()
          cleared = !after.invariants.some(
            (i) => !i.holds && i.heal === 'repair' && failingRepairs.includes(i.name)
          )
        } catch {
          // The post-heal re-read failing is itself reportable, never a crash.
        }
        record(`provider:${report.provider}`, {
          checked: true,
          healed: cleared ? failingRepairs.length : 0,
          detail: cleared
            ? `incremental repair cleared: ${failingRepairs.join(', ')}`
            : `repair() ran but the re-read still fails (${
                after
                  ? after.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')
                  : 're-read threw'
              }) — escalate to repairIndex({ rebuild: ['${familyName}'] })`,
          reason: cleared ? undefined : 'repair did not converge'
        })
      } else {
        record(`provider:${report.provider}`, {
          checked: true, healed: 0,
          detail: `unhealthy without a routable verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})`
        })
      }
    }
    // detectAndRepairCorruption() above rebuilt the derived indexes from
    // canonical, so any adopt-forward degraded ids and a non-fatal init
    // rebuild failure are now reconciled — clear the queryable degraded state
    // and re-arm the read-path warning.
    if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) {
      beginPhase('degraded-read-state', 'clearing degraded ids and re-arming the read-path warning')
      this._indexDegradedIds.clear()
      this._indexRebuildFailed = null
      this._degradedReadWarned = false
      record('degraded-read-state', { checked: true, healed: 1, detail: 'degraded ids cleared, read-path warning re-armed' })
    }

    const healedTotal = families.reduce((n, f) => n + f.healed, 0)
    const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt }
    prodLog.narrate(
      `[Brainy] repairIndex complete in ${report.durationMs}ms — ` +
        `${families.filter((f) => f.checked).length}/${families.length} families checked, ` +
        `${healedTotal} heal(s): ` +
        families
          .map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}@${f.durationMs ?? 0}ms`)
          .join(', ')
    )
    return report
  }

  /**
   * Register a plugin manually.
   *
   * Must be called BEFORE init(). Plugins registered after init()
   * will not be activated.
   */
  use(plugin: BrainyPlugin): this {
    this.pluginRegistry.register(plugin)
    return this
  }

  /**
   * Get list of active plugin names.
   */
  getActivePlugins(): string[] {
    return this.pluginRegistry.getActivePlugins()
  }

  /**
   * Auto-detect and activate plugins.
   * Called internally during init().
   */
  /**
   * Import a plugin package by name. Isolated as a seam so tests can simulate
   * the three auto-detect outcomes (not installed / broken install / valid)
   * without the package being present. The specifier is a variable, so
   * bundlers cannot statically resolve — and cannot force-include — the
   * optional accelerator.
   */
  private async importPluginPackage(pkg: string): Promise<any> {
    return import(pkg)
  }

  /**
   * True when a dynamic-import failure means "the package itself is not
   * installed" (the silent free path), as opposed to "the package is present
   * but broken" (which must fail loud). Node and Bun both name the missing
   * package in the resolution error; a failure naming anything else — an
   * internal file, a dependency of the plugin, a syntax error — is a broken
   * install, never a not-installed.
   */
  private static isPackageNotInstalledError(error: unknown, pkg: string): boolean {
    const code = (error as { code?: string })?.code
    const message = error instanceof Error ? error.message : String(error)
    // The package name must TERMINATE where it ends: an unanchored prefix match
    // read a missing platform-binary SIBLING package (e.g. "<pkg>-linux-x64-gnu",
    // exactly what a deploy replacing node_modules mid-restart leaves behind) as
    // "<pkg> is not installed" — and a present-but-broken accelerator silently
    // degraded to the default JS engines. A production storm was hunted for a
    // day because of that swallow. The name must be followed by a quote,
    // whitespace, punctuation, or end-of-message — never a longer name's tail.
    const escaped = pkg.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
    const namesPackage = new RegExp("(^|['\"\\s])" + escaped + "(?=$|['\"\\s.,)])").test(message)
    const isResolutionFailure =
      code === 'ERR_MODULE_NOT_FOUND' ||
      code === 'MODULE_NOT_FOUND' ||
      /cannot find (package|module)/i.test(message) ||
      /failed to resolve/i.test(message) // Bun's resolver phrasing
    return isResolutionFailure && namesPackage
  }

  private async loadPlugins(): Promise<void> {
    // plugins config:
    //   undefined (default) → guarded auto-detection of the first-party
    //                         accelerator: installing @soulcraft/cor IS the
    //                         opt-in. Not installed → plain brainy, silently.
    //                         Installed → it loads and announces itself; if it
    //                         is installed but broken, init() THROWS — an
    //                         installed accelerator never silently vanishes.
    //   false / []          → no plugins, no detection (explicit opt-out)
    //   ['@soulcraft/cor']  → load only these explicitly listed packages
    // Note: plugins registered via brain.use() are always activated regardless of config
    const pluginConfig = this.config.plugins
    if (Array.isArray(pluginConfig) && pluginConfig.length > 0) {
      // Explicit list: import and register the specified packages. A package
      // listed here is REQUIRED — a missing or invalid plugin must fail LOUD,
      // never silently fall back to the default engine (the cross-repo drift
      // this whole guard exists to prevent).
      for (const pkg of pluginConfig) {
        let mod: any
        try {
          mod = await this.importPluginPackage(pkg)
        } catch (error) {
          throw new Error(
            `[brainy] Plugin "${pkg}" is listed in config.plugins but could not be loaded: ` +
              `${error instanceof Error ? error.message : String(error)}. ` +
              `Install it (e.g. npm i ${pkg}) or remove it from config.plugins.`
          )
        }
        const plugin: BrainyPlugin = mod.default || mod
        if (plugin && typeof plugin.activate === 'function' && plugin.name) {
          this.pluginRegistry.register(plugin)
        } else {
          throw new Error(
            `[brainy] Package "${pkg}" (config.plugins) is not a valid Brainy plugin ` +
              `(missing { name, activate }). Remove it from config.plugins.`
          )
        }
      }
    } else if (pluginConfig === undefined) {
      // Guarded auto-detection (default). Installing the first-party
      // accelerator is the opt-in — probe for it, and apply the SAME loud
      // posture as the explicit branch to everything except "not installed":
      // a present-but-broken accelerator must never silently degrade to JS.
      for (const pkg of Brainy.AUTO_DETECT_PLUGIN_PACKAGES) {
        let mod: any
        try {
          mod = await this.importPluginPackage(pkg)
        } catch (error) {
          if (Brainy.isPackageNotInstalledError(error, pkg)) {
            continue // the free path: not installed, nothing to load, no noise
          }
          throw new Error(
            `[brainy] The accelerator "${pkg}" is installed but failed to load: ` +
              `${error instanceof Error ? error.message : String(error)}. ` +
              `brainy will NOT silently run the default JS engines in its place — ` +
              `fix the install (npm i ${pkg}) or disable detection with plugins: [].`
          )
        }
        const plugin: BrainyPlugin = mod.default || mod
        if (plugin && typeof plugin.activate === 'function' && plugin.name) {
          this.pluginRegistry.register(plugin)
        } else {
          throw new Error(
            `[brainy] The installed accelerator "${pkg}" is not a valid Brainy plugin ` +
              `(missing { name, activate }) — a broken or incompatible install. ` +
              `brainy will NOT silently run the default JS engines in its place — ` +
              `fix the install (npm i ${pkg}) or disable detection with plugins: [].`
          )
        }
      }
    }

    // Create plugin context
    const context: BrainyPluginContext = {
      registerProvider: (key, impl) => this.pluginRegistry.registerProvider(key, impl),
      version: getBrainyVersion()
    }

    // Activate all registered plugins
    const activated = await this.pluginRegistry.activateAll(context)

    // Version-coupling guard (the provider-key cliff). A pre-8.0 native plugin
    // (e.g. @soulcraft/cor < 3.0) registers its vector engine under a LEGACY key
    // ('hnsw'/'diskann') instead of the 8.0 canonical 'vector'. brainy 8.x reads
    // only 'vector', so without this check the native engine silently never
    // engages and every query runs on the default JS index — exactly the
    // invisible degrade we must prevent. brainy never registers these keys
    // itself, so their presence can only come from a plugin.
    const LEGACY_VECTOR_KEYS = ['hnsw', 'diskann', 'hnswIndex']
    const legacyVector = LEGACY_VECTOR_KEYS.filter(
      (k) => this.pluginRegistry.getProvider(k) !== undefined
    )
    if (legacyVector.length > 0 && this.pluginRegistry.getProvider('vector') === undefined) {
      throw new Error(
        `[brainy] A native acceleration plugin registered a legacy vector provider ` +
          `(${legacyVector.join(', ')}) but not the 'vector' provider that brainy 8.x requires. ` +
          `This is an incompatible (pre-8.0) accelerator — upgrade to @soulcraft/cor >= 3.0. ` +
          `brainy will NOT silently run the default JS vector engine in its place.`
      )
    }

    if (activated.length > 0) {
      // Only log if not in silent mode
      if (!this.config.silent) {
        for (const name of activated) {
          console.log(`[brainy] Plugin activated: ${name}`)
        }
      }
    }
  }

  /**
   * Execute an aggregate query, returning results as Result<T>[] for API consistency.
   */
  private async findAggregate(params: FindParams<T>): Promise<Result<T>[]> {
    if (!this._aggregationIndex) {
      throw new Error('No aggregates defined. Call defineAggregate() first.')
    }

    // Normalize aggregate params
    const aggParams: AggregateQueryParams = typeof params.aggregate === 'string'
      ? { name: params.aggregate }
      : params.aggregate as AggregateQueryParams

    // Merge find-level params into aggregate query
    if (params.where && !aggParams.where) {
      aggParams.where = params.where as Record<string, unknown>
    }
    if (params.orderBy && !aggParams.orderBy) {
      aggParams.orderBy = params.orderBy
    }
    if (params.order && !aggParams.order) {
      aggParams.order = params.order
    }
    if (params.limit !== undefined && aggParams.limit === undefined) {
      aggParams.limit = params.limit
    }
    if (params.offset !== undefined && aggParams.offset === undefined) {
      aggParams.offset = params.offset
    }

    // Backfill from already-stored entities if this aggregate was defined over a
    // populated store (write-time hooks only capture entities added after define).
    await this.backfillAggregateIfNeeded(aggParams.name)

    const aggregateResults = this._aggregationIndex.queryAggregate(aggParams)

    // Convert AggregateResult[] to Result<T>[] for API consistency
    return aggregateResults.map((agg, index) => {
      const entity: Entity<T> = {
        id: agg.entityId || `__agg_${aggParams.name}_${index}`,
        vector: [],
        type: NounType.Measurement,
        data: `${aggParams.name}: ${Object.entries(agg.groupKey).map(([k, v]) => `${k}=${v}`).join(', ')}`,
        metadata: {
          ...agg.groupKey,
          ...agg.metrics,
          __aggregate: aggParams.name,
          count: agg.count
        } as T,
        createdAt: Date.now(),
        service: 'brainy:aggregation'
      }

      return {
        id: entity.id,
        score: 1.0,
        type: NounType.Measurement,
        metadata: entity.metadata,
        data: entity.data,
        entity,
        // Surface the documented AggregateResult fields at the top level so consumers can read
        // groupKey/metrics/count directly. Previously these were only reachable under .metadata,
        // so callers expecting an AggregateResult saw rows with no groupKey/metrics/count and
        // interpreted the output as degenerate/empty.
        groupKey: agg.groupKey,
        metrics: agg.metrics,
        count: agg.count
      }
    })
  }

  /**
   * Backfill a named aggregate from entities already in storage.
   *
   * Write-time hooks (`onEntityAdded` etc.) only capture entities added *after* an
   * aggregate is defined, so an aggregate defined over a populated store — the common
   * case under durable storage, where a brain reopens pre-populated — would otherwise
   * return `[]`. On first query we clear the aggregate's state and stream every stored
   * noun back through it. Storage-agnostic: `getNouns()` works for in-memory, the
   * filesystem adapter, and native (Cor) storage alike. One-time per definition —
   * the rebuilt state is persisted on flush() and reloaded on the next session.
   */
  private async backfillAggregateIfNeeded(name: string): Promise<void> {
    const index = this._aggregationIndex
    if (!index) return

    // Persisted-state adoption happens inside ready(); after it resolves the
    // pending-backfill set is authoritative (an unchanged definition with valid
    // persisted state is NOT listed — no walk at all on a clean reopen).
    await index.ready()

    // Behind-stamp catch-up FIRST (SELF-ENGINE-LIFECYCLE-SPRINT ask (b)):
    // adopted-but-behind state reconciles its exact missing window
    // incrementally — bounded by that window's affected entities — instead
    // of the whole-store rescan an unclean exit used to force. Single-flight
    // like the walk below; a failed catch-up demotes to a LOUD rescan.
    if (index.getPendingCatchUps().length > 0) {
      if (!this._aggregationCatchUpFlight) {
        this._aggregationCatchUpFlight = this.runAggregationCatchUp().finally(() => {
          this._aggregationCatchUpFlight = null
        })
      }
      await this._aggregationCatchUpFlight
    }

    // Single-flight: concurrent queries share ONE walk instead of each wiping
    // the others' partial state and starting their own (the stampede that kept
    // a busy store from ever converging). The loop covers the rare case where
    // the in-flight walk snapshotted its batch before `name` became pending —
    // the next iteration starts a fresh walk that includes it.
    while (index.getPendingBackfills().includes(name)) {
      // Failure latch: a deterministically-failing walk must not be re-run at
      // the caller's retry rate — that is a silent CPU loop wearing a retry
      // loop's clothes. Within the cooldown, rethrow the recorded failure
      // immediately; after it, one fresh attempt is allowed.
      const failure = this._aggregationBackfillFailure
      if (failure && Date.now() - failure.at < AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS) {
        throw new Error(
          `Aggregation backfill for '${name}' is in failure cooldown (retry in ` +
            `${Math.ceil((AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS - (Date.now() - failure.at)) / 1000)}s). ` +
            `Last failure: ${failure.error.message}`
        )
      }
      if (!this._aggregationBackfillFlight) {
        this._aggregationBackfillFlight = this.runAggregationBackfillWalk()
          .finally(() => {
            this._aggregationBackfillFlight = null
          })
      }
      await this._aggregationBackfillFlight
    }
  }

  /**
   * @description Build the aggregation view of a LIVE entity — top-level
   * engine fields + the user bag, the same shape `entityForIndexing` and
   * `entityForAggFromRawRecord` produce, so group keys and source filters
   * resolve identically whichever door an entity arrives through.
   */
  private aggViewFromEntity(e: Entity<T>): Record<string, unknown> {
    return {
      type: e.type,
      ...(e.subtype !== undefined && { subtype: e.subtype }),
      ...((e as unknown as Record<string, unknown>).visibility !== undefined && {
        visibility: (e as unknown as Record<string, unknown>).visibility
      }),
      ...(e.confidence !== undefined && { confidence: e.confidence }),
      ...(e.weight !== undefined && { weight: e.weight }),
      createdAt: e.createdAt,
      updatedAt: e.updatedAt,
      ...(e.service !== undefined && { service: e.service }),
      ...(e.data !== undefined && { data: e.data }),
      ...(e.createdBy !== undefined && { createdBy: e.createdBy }),
      metadata: e.metadata ?? {}
    }
  }

  /** Cap on a catch-up window's affected-entity count before demoting to a rescan. */
  private static readonly AGGREGATION_CATCHUP_MAX_AFFECTED = 5000

  /**
   * Reconcile every behind-stamp aggregate's exact missing window
   * `(from, to]` using the fact log for the AFFECTED ID SET and time-travel
   * reads for exact before/after states — cost bounded by writes since the
   * last flush, never store size. Reconciliation targets the FIXED window
   * end (`to` = the committed generation at adoption), so live write hooks
   * compose exactly: every application on both paths is a precise old/new
   * delta pair, and interleaving cannot drift totals. Any failure or an
   * oversized window demotes to the announced full rescan — never a silent
   * partial serve.
   */
  private async runAggregationCatchUp(): Promise<void> {
    const index = this._aggregationIndex!
    const catchups = index.getPendingCatchUps()
    if (catchups.length === 0) return

    const startedAt = Date.now()
    try {
      // One fact scan covers every window (they share flush boundaries in
      // practice); per-name windows filter per id below.
      const from = Math.min(...catchups.map(c => c.from))
      const to = Math.max(...catchups.map(c => c.to))
      const scan = this.scanFacts({ fromGeneration: from + 1, toGeneration: to, kinds: ['noun'] })
      if (!scan) {
        for (const c of catchups) {
          index.demoteCatchUpToBackfill(c.name, 'no fact log on this store — window unreadable')
        }
        return
      }

      // id → generations it changed at, inside the union window.
      const affected = new Map<string, number[]>()
      for await (const batch of scan.batches()) {
        for (const fact of batch.facts) {
          for (const op of fact.ops) {
            if (op.kind !== 'noun') continue
            const gens = affected.get(op.id)
            if (gens) gens.push(fact.generation)
            else affected.set(op.id, [fact.generation])
          }
        }
        if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) break
      }
      if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) {
        for (const c of catchups) {
          index.demoteCatchUpToBackfill(
            c.name,
            `window touches >${Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED} entities — a rescan is cheaper`
          )
        }
        return
      }

      // Exact before/after views per unique generation bound, via time travel.
      const dbCache = new Map<number, Db<T>>()
      const dbAt = async (gen: number): Promise<Db<T>> => {
        let db = dbCache.get(gen)
        if (!db) {
          db = await this.asOf(gen)
          dbCache.set(gen, db)
        }
        return db
      }
      try {
        for (const c of catchups) {
          const beforeDb = await dbAt(c.from)
          const afterDb = await dbAt(c.to)
          let reconciled = 0
          for (const [id, gens] of affected) {
            if (!gens.some(g => g > c.from && g <= c.to)) continue
            const [before, after] = await Promise.all([beforeDb.get(id), afterDb.get(id)])
            index.reconcileEntity(
              c.name,
              id,
              before ? this.aggViewFromEntity(before) : null,
              after ? this.aggViewFromEntity(after) : null
            )
            reconciled++
          }
          index.finishCatchUp(c.name)
          prodLog.info(
            `[Aggregation] '${c.name}': caught up generations ${c.from}→${c.to} — ` +
              `${reconciled} entit${reconciled === 1 ? 'y' : 'ies'} reconciled in ${Date.now() - startedAt}ms (no store rescan)`
          )
        }
      } finally {
        await Promise.all(Array.from(dbCache.values(), db => db.release().catch(() => {})))
      }
    } catch (err) {
      for (const c of index.getPendingCatchUps()) {
        index.demoteCatchUpToBackfill(c.name, `catch-up failed: ${(err as Error).message}`)
      }
    }
  }

  /**
   * One store walk fills EVERY aggregate currently pending backfill — M pending
   * aggregates cost one enumeration, not M. Only reached when an aggregate
   * genuinely needs a rescan (new definition over a populated store, changed
   * definition, or failed state load); a clean reopen adopts persisted state
   * and never walks.
   */
  private async runAggregationBackfillWalk(): Promise<void> {
    const index = this._aggregationIndex!
    const names = index.getPendingBackfills()
    if (names.length === 0) return

    prodLog.info(`[Aggregation] backfill walk starting for: ${names.join(', ')}`)
    const startedAt = Date.now()
    for (const n of names) index.beginBackfill(n)

    // SELF-ENGINE-LIFECYCLE-SPRINT ask (c): when the native provider offers
    // the parallel whole-rebuild (`rebuildAggregate` — on the contract since
    // 8.x but never invoked), collect the walk's views and hand them over in
    // ONE call per aggregate instead of a per-entity FFI stream. Memory note:
    // the collected views are metadata-only records (no vectors); at the
    // scales where this walk is even reached the array is the cheap part —
    // the per-entity FFI round-trips were the measured cost.
    const useProviderRebuild = index.hasProviderRebuild()
    const collected: Array<Record<string, unknown>> = []

    let scanned = 0
    try {
      const PAGE = 500
      let offset = 0
      let cursor: string | undefined
      for (;;) {
        const page = await this.storage.getNouns({
          pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
        })
        for (const noun of page.items) {
          const record = noun as unknown as Record<string, unknown>
          if (useProviderRebuild) {
            collected.push(record)
          } else {
            for (const n of names) {
              index.backfillEntity(n, record)
            }
          }
        }
        scanned += page.items.length
        if (!page.hasMore || page.items.length === 0) break
        if (page.nextCursor) {
          if (page.nextCursor === cursor) {
            // A non-advancing cursor with hasMore=true would loop this walk at
            // CPU speed forever, silently. That is a storage pagination defect —
            // fail the waiting queries loudly instead of spinning.
            throw new Error(
              `Aggregation backfill aborted: storage pagination returned a non-advancing cursor ` +
                `after ${scanned} entities with hasMore=true — the storage adapter's getNouns cursor is broken.`
            )
          }
          cursor = page.nextCursor
        } else {
          offset += page.items.length
        }
      }
    } catch (err) {
      // Non-destructive failure: drop the staging maps (live state keeps
      // serving), keep the aggregates flagged pending, latch the error so
      // retries within the cooldown fail fast, and say all of it out loud.
      for (const n of names) index.abortBackfill(n)
      this._aggregationBackfillFailure = { at: Date.now(), error: err as Error }
      prodLog.warn(
        `[Aggregation] backfill walk FAILED after ${scanned} entities: ${(err as Error).message} — ` +
          `prior aggregate state preserved; retries suppressed for ${AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS / 1000}s`
      )
      throw err
    }

    if (useProviderRebuild) {
      for (const n of names) {
        if (!index.rebuildWithProvider(n, collected)) {
          // Provider refused/absent for this one — stream it the JS way.
          for (const record of collected) index.backfillEntity(n, record)
          index.finishBackfill(n)
        }
      }
    } else {
      for (const n of names) index.finishBackfill(n)
    }
    this._aggregationBackfillFailure = null
    prodLog.info(
      `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) ` +
        `in ${Date.now() - startedAt}ms${useProviderRebuild ? ' (native parallel rebuild)' : ''}`
    )
  }

  /**
   * @description Close and clean up: flush every buffered component, stamp
   * the durability markers, release resources, then give up the writer lock.
   *
   * TWO PARTS, AND THE SECOND IS UNCONDITIONAL. Everything that persists data
   * runs in {@link closeDurableSteps}; the terminal releases — the flush-request
   * watcher, the WRITER LOCK, the VFS timers, and the terminal `closed` flag —
   * run whether those steps succeeded or not, in a `finally`. A close that
   * threw halfway used to strand the writer lock on disk with this process's
   * (soon dead) pid in it, so the next boot of every affected store announced
   * `Overwriting stale writer lock … appears dead` after an orderly exit and
   * an operator had to decide whether their database had crashed. A closed
   * brain holds no lock — there is no failure for which the opposite is the
   * safer answer.
   *
   * The original failure is never swallowed: it is narrated with what it costs
   * the next open, then rethrown to the caller.
   * @returns Nothing.
   * @throws The first failure from the durable close steps, after the
   *   terminal releases have run.
   */
  async close(): Promise<void> {
    let closeFailure: unknown = null
    try {
      await this.closeDurableSteps()
    } catch (error) {
      closeFailure = error
    }

    // ---- TERMINAL RELEASES: always, even after a failure above ----

    // Stop the cross-process flush-request watcher (no-op if never started).
    try {
      if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') {
        this.storage.stopFlushRequestWatcher()
      }
    } catch (error) {
      console.warn('[Brainy] close: stopping the flush-request watcher failed:', error)
    }

    // Release the writer lock. Runs after the metadata buffer drain in
    // closeDurableSteps() — otherwise a pending write could land after a
    // successor writer claimed the lock — and runs even if that drain threw:
    // holding a lock from a process that is about to exit locks the store's
    // next boot out of a clean verdict.
    try {
      if (this.storage && typeof this.storage.releaseWriterLock === 'function') {
        await this.storage.releaseWriterLock()
      }
    } catch (error) {
      console.warn('[Brainy] close: releasing the writer lock failed:', error)
    }

    // Shut down the VFS: stops its background maintenance interval and the
    // PathResolver's — both are ref'd timers that would keep the process
    // alive after the last brain closes (consumer-reported hang).
    try {
      if (this._vfs) {
        await this._vfs.close()
      }
    } catch (error) {
      console.warn('[Brainy] close: VFS shutdown failed:', error)
    }

    this.initialized = false
    // close() is terminal: block lazy re-initialization on any subsequent
    // operation (ensureInitialized() throws once this is set). Set even when
    // the durable steps failed — a half-closed brain must not keep serving.
    this.closed = true

    // Drop this instance from the global registry, and when it was the last
    // one, deregister the global shutdown hooks — their ref'd signal handles
    // would otherwise keep the process alive after every brain is closed.
    const instanceIndex = Brainy.instances.indexOf(this)
    if (instanceIndex !== -1) {
      Brainy.instances.splice(instanceIndex, 1)
    }
    Brainy.deregisterShutdownHooksIfIdle()

    if (closeFailure !== null) {
      console.error(
        `[Brainy] close FAILED partway: ` +
          `${closeFailure instanceof Error ? closeFailure.message : String(closeFailure)}\n` +
          `  This brain is closed and holds no writer lock, but the clean-shutdown ` +
          `marker may not have been written — the next open will run crash recovery ` +
          `(a generation-log fold) and report its wall.`
      )
      throw closeFailure
    }
  }

  /**
   * @description The durable half of {@link close}: flush every component,
   * persist the generation counter and its markers, close the components,
   * deactivate plugins, drain the metadata write buffer. Separated from
   * `close()` so the terminal releases there can run in a `finally` — see that
   * method's contract.
   * @returns Nothing.
   */
  private async closeDurableSteps(): Promise<void> {
    // Persistence cadence teardown: no background flush may fire after close
    // begins (close() runs its own final flush).
    if (this._persistIdleTimer) {
      clearTimeout(this._persistIdleTimer)
      this._persistIdleTimer = null
    }
    if (this._persistBackgroundFlight) {
      await this._persistBackgroundFlight.catch(() => {})
    }

    // Cancel any pending post-import background deduplication FIRST — it is a
    // writer (merge-deletes), and no delete pass may start mid- or post-close.
    this._backgroundDedup?.cancelPending()

    // Change-feed teardown: no events are delivered for or after close().
    this._changeFeed.close()

    // Phase 0a: Persist buffered single-op generation history (async
    // group-commit) before anything else, so a clean close never drops history
    // the caller already observed. No-op when nothing is pending or read-only.
    if (this.generationStore && !this.isReadOnly) {
      await this.generationStore.flushPendingSingleOps()
    }

    // Phase 0b: REPACK cold history into sealed segments (D1+D3 —
    // re-representation, never deletion; the only history transform under the
    // archival profile), then auto-compact per config.retention. Repack runs
    // FIRST so bounded-retention reclaim can drop whole segments. Both are
    // time-bounded maintenance passes (8.9.0 law: flush() never pays these);
    // both are housekeeping — failures warn, never fail a clean shutdown.
    if (!this.isReadOnly && this.generationStore) {
      try {
        await this.generationStore.repackHistory({ timeBudgetMs: 5_000 })
      } catch (error) {
        console.warn(
          `History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}`
        )
      }
    }
    await this.autoCompactHistory()

    // Watermark stamps ride this flush too — see stampProjectionWatermarks().
    // Read-only instances skip it (no writes, no committed-generation drift
    // to certify; ensureInitialized()'s guard below never runs for them
    // either, so this must not assume a writer's invariants).
    if (!this.isReadOnly) {
      this.stampProjectionWatermarks()
    }

    // Phase 1: Flush ALL components in parallel to persist buffered data
    // This is critical when cor native providers buffer data in Rust memory
    await Promise.all([
      // Flush HNSW dirty nodes (deferred persistence mode)
      (async () => {
        if (this.index && typeof this.index.flush === 'function') {
          await this.index.flush()
        }
      })(),
      // Flush metadata index (field indexes + EntityIdMapper)
      (async () => {
        if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') {
          await this.metadataIndex.flush()
        }
      })(),
      // Flush graph adjacency index (LSM trees)
      (async () => {
        if (this.graphIndex && typeof this.graphIndex.flush === 'function') {
          await this.graphIndex.flush()
        }
      })(),
      // Flush storage adapter counts
      (async () => {
        if (this.storage && typeof this.storage.flushCounts === 'function') {
          await this.storage.flushCounts()
        }
      })(),
      // Flush aggregation index state
      (async () => {
        if (this._aggregationIndex) {
          await this._aggregationIndex.flush()
        }
      })(),
      // 8.0 MVCC: detach the generation-bump hook and persist the counter
      (async () => {
        if (this.generationStore) {
          await this.generationStore.close()
        }
      })()
    ])

    // Stamp the entity tree at the close boundary (counters + counter now
    // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT
    // instead of benign-behind. Best-effort, never blocks the close.
    if (this.generationStore && !this.isReadOnly) {
      await this.stampEntityTree()
    }

    // Phase 2: Close components to release resources (timers, file handles)
    // Data is already safe on disk from Phase 1
    await Promise.all([
      (async () => {
        if (this.graphIndex && typeof this.graphIndex.close === 'function') {
          await this.graphIndex.close()
        }
      })(),
      (async () => {
        const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks
        if (index && typeof index.close === 'function') {
          await index.close()
        }
      })(),
      (async () => {
        const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks
        if (metadataIndex && typeof metadataIndex.close === 'function') {
          await metadataIndex.close()
        }
      })(),
      (async () => {
        if (this._materializer) {
          this._materializer.close()
        }
      })()
    ])

    // Deactivate plugins (safe — all data flushed and resources released above)
    await this.pluginRegistry.deactivateAll()

    // Restore console methods if silent mode was enabled
    if (this.config.silent && this.originalConsole) {
      console.log = this.originalConsole.log as typeof console.log
      console.info = this.originalConsole.info as typeof console.info
      console.warn = this.originalConsole.warn as typeof console.warn
      console.error = this.originalConsole.error as typeof console.error
      this.originalConsole = undefined
    }

    // Drain the metadata write buffer if the storage adapter has one
    if (this.storage && 'metadataWriteBuffer' in this.storage) {
      // Boundary: drains BaseStorage's protected `metadataWriteBuffer` member
      // (cloud adapters initialize it in their init()). No public drain hook
      // exists on the adapter surface, and close() must not leave a pending
      // write to land after a successor writer claims the lock.
      const buffer = (this.storage as unknown as {
        metadataWriteBuffer?: MetadataWriteBuffer | null
      }).metadataWriteBuffer
      if (buffer && typeof buffer.destroy === 'function') {
        await buffer.destroy()
      }
    }

  }
}

/**
 * @description Whether a `where` clause actually constrains the result set —
 * i.e. it is a non-null object carrying at least one predicate key. An empty
 * `where: {}` carries ZERO predicates and must behave exactly like an absent
 * `where` everywhere it is consulted; treating it as "a filter is present"
 * routes the query into the index-filter path, where `getIdsForFilter({})`
 * answers `[]` by contract — a silent empty on a match-all query (the
 * forbidden answer class: served-or-refused, never silently nothing).
 * @param where - The raw `where` value from a query/selector params object.
 * @returns `true` when `where` holds at least one predicate.
 */
function whereConstrains(where: unknown): where is Record<string, unknown> {
  return (
    where !== null &&
    typeof where === 'object' &&
    !Array.isArray(where) &&
    Object.keys(where).length > 0
  )
}

/**
 * @description Extract the entity/relationship id from a canonical storage
 * path of the form `entities/(nouns|verbs)/<shard>/<id>/metadata.json`.
 * Vector-file and non-canonical paths return `null` — the historical
 * materializer enumerates each id exactly once, from its metadata file.
 * @param path - A storage-root-relative object path.
 * @returns The id, or `null` when the path is not a metadata file.
 */
function entityIdFromCanonicalPath(path: string): string | null {
  const match = /^entities[/\\](?:nouns|verbs)[/\\][^/\\]+[/\\]([^/\\]+)[/\\]metadata\.json$/.exec(path)
  return match ? match[1] : null
}

/**
 * @description Narrow `BrainyConfig['storage']` to a pre-constructed storage
 * adapter instance (vs. a factory config object). Instances are detected by
 * their `init` method — config objects are plain data and never carry one.
 * @param value - The configured storage value.
 * @returns Whether `value` is an adapter instance to use directly.
 */
function isStorageAdapterInstance(
  value: BrainyConfig['storage']
): value is StorageAdapter {
  return !!value && typeof (value as StorageAdapter).init === 'function'
}

// Re-export types for convenience
export * from './types/brainy.types.js'
export { NounType, VerbType } from './types/graphTypes.js'