fix: full-removal canonical deletes + family-scoped migration gate

Two production-reported spine fixes:

- Canonical delete is a FULL removal: remove()/removeMany() deleted the
  metadata leg but never the canonical vectors.json leg or the <id>/
  directory, leaving ghost rows that inflated enumerated counts forever,
  read as damage scars, and caused duplicate readdir entries for
  re-created VFS paths (the stale-unpost path). The delete operation now
  routes through storage.deleteNoun — both legs + the entity container —
  with a full two-leg before-image rollback; deleteVerb symmetric; the
  blind 'no metadata file' catch that masked real faults is gone. The
  generation log still holds the delete's before-image, so asOf() history
  is unchanged. repairIndex() gains a conservative, loud orphan-prune
  sweep (containers with no metadata content leg) + count recompute for
  stores damaged by earlier versions.

- Family-scoped migration gate: every read blocked on the whole-brain
  migration lock even when the migrating index was irrelevant. The gate
  now scopes to the index families a read actually consults — canonical
  reads never wait; find() waits only on its query shape's families;
  graph traversals wait only on the graph family. Writes keep the
  conservative whole-brain wait. A read that needs the migrating family
  still blocks (bounded) with the retryable MigrationInProgressError.

src/brainy.ts

@@ -367,6 +367,15 @@ class InsertPreconditionExistsSignal extends Error {
   }
 }
 
+/**
+ * @description The derived-index families a read may depend on. A read that
+ * consults none of them (a canonical-storage read: `get`, an entity
+ * enumeration, a VFS content/dir read) is index-independent and must never
+ * block on another family's one-time migration. Used by the family-scoped
+ * migration gate ({@link Brainy.awaitMigrationLock}).
+ */
+export type IndexFamily = 'vector' | 'metadata' | 'graph'
+
 /**
  * The main Brainy class - Clean, Beautiful, Powerful
  * REAL IMPLEMENTATION - No stubs, no mocks
@@ -1439,7 +1448,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init
    * opportunity.
    */
-  private async ensureInitialized(opts?: { bypassMigrationLock?: boolean }): Promise<void> {
+  private async ensureInitialized(opts?: {
+    bypassMigrationLock?: boolean
+    needs?: IndexFamily[]
+  }): Promise<void> {
     if (this.closed) {
       throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.')
     }
@@ -1449,12 +1461,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // Coordinated migration LOCK (#18): every data-plane read and write funnels
     // through here, so this is the single choke point that holds operations while
     // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no
-    // op touches a half-built index. Observability (`health`/`checkHealth`) and
-    // the lock-clearing path (`stampBrainFormat`, which does not route through
-    // here) opt out so an operator can always watch progress and cor can stamp.
+    // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the
+    // derived-index families this operation actually consults, so a read served
+    // entirely from canonical storage (`needs: []`) or from a healthy family
+    // never blocks on an UNRELATED family's migration. `needs` omitted = the
+    // conservative whole-brain wait (writes, and any read not yet classified).
+    // Observability (`health`/`checkHealth`) and the lock-clearing path
+    // (`stampBrainFormat`, which does not route through here) opt out entirely so
+    // an operator can always watch progress and a native provider can stamp.
     // A brain that never migrates pays one boolean check (see awaitMigrationLock).
     if (!opts?.bypassMigrationLock) {
-      await this.awaitMigrationLock()
+      await this.awaitMigrationLock(opts?.needs)
     }
   }
 
@@ -2188,7 +2205,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    *
    */
   async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
-    await this.ensureInitialized()
+    // Canonical read: a get resolves an entity by id straight from storage and
+    // consults no derived index — it must not wait on any family's migration.
+    await this.ensureInitialized({ needs: [] })
     this.warnIfReadsDegraded('get')
 
     // Id normalization (8.0): a caller may read by their natural key — resolve
@@ -2247,7 +2266,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * ```
    */
   async batchGet(ids: string[], options?: GetOptions): Promise<Map<string, Entity<T>>> {
-    await this.ensureInitialized()
+    // Canonical read (see get): resolves by id from storage, no derived index.
+    await this.ensureInitialized({ needs: [] })
 
     // Id normalization (8.0): resolve each id to its canonical UUID so callers
     // can batch-read by natural key. resolveEntityId is idempotent on real
@@ -4271,7 +4291,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   async related(
     paramsOrId?: string | RelatedParams
   ): Promise<Relation<T>[]> {
-    await this.ensureInitialized()
+    // Graph read: relationship traversal consults only the graph adjacency
+    // family, so it waits on a graph migration but not on vector/metadata.
+    await this.ensureInitialized({ needs: ['graph'] })
     await this.verifyGraphAdjacencyLive()
 
     // Handle string ID shorthand: related(id) -> related({ from: id })
@@ -5810,7 +5832,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * })
    */
   async find(query: string | FindParams<T>): Promise<Result<T>[]> {
-    await this.ensureInitialized()
+    // Init only here — the migration gate is applied family-scoped below, once
+    // the query params reveal which index families this read actually consults
+    // (a string query may parse to a where / connected / vector shape). The lazy
+    // loader and cold-read probes below already defer to a migrating provider.
+    await this.ensureInitialized({ needs: [] })
 
     // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true)
     // This is a production-safe, concurrency-controlled lazy load
@@ -5849,6 +5875,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // Zero-config validation (static import for performance)
     validateFindParams(params)
 
+    // Family-scoped migration gate: wait only on the index families THIS query
+    // consults, so a filter or graph query served from a healthy family never
+    // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once
+    // params are final (after natural-language parse + connected-id resolution)
+    // and before the aggregate path, which carries no gate of its own.
+    await this.awaitMigrationLock(this.queryIndexFamilies(params))
+
     // Aggregate query path — early return when params.aggregate is set
     if (params.aggregate) {
       return this.findAggregate(params)
@@ -12530,14 +12563,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     entityCount: number
     indexEntryCount: number
     recommendation: string | null
-    /** Cross-layer (ADR-004 §6): each provider's own invariant self-report, when
+    /** Cross-layer: each provider's own invariant self-report, when
      *  it exposes validateInvariants(). Absent providers are simply not listed. */
     providers?: ProviderInvariantReport[]
   }> {
     await this.ensureInitialized()
     const result = await this.metadataIndex.validateConsistency()
 
-    // Cross-layer integrity (ADR-004 §6): validateConsistency() above only sees the
+    // Cross-layer integrity: validateConsistency() above only sees the
     // JS metadata index — it is BLIND to a native provider whose manifest ↔
     // segments ↔ counts have diverged. Delegate to each provider's own
     // validateInvariants() and aggregate, so "healthy-while-broken" is impossible.
@@ -12583,7 +12616,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
 
   /**
    * @description Feature-detect + call each index provider's OPTIONAL
-   * `validateInvariants()` (ADR-004 §6) and collect the reports. The contract is
+   * `validateInvariants()` and collect the reports. The contract is
    * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a
    * provider violates that, the throw is turned into an UNHEALTHY synthetic
    * report (loud), never swallowed into "healthy". Providers that do not expose
@@ -12673,7 +12706,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       limit?: number
     }
   ): Promise<string[]> {
-    await this.ensureInitialized()
+    // Graph read (see related): adjacency traversal only.
+    await this.ensureInitialized({ needs: ['graph'] })
     await this.verifyGraphAdjacencyLive()
 
     const direction = options?.direction || 'both'
@@ -14844,6 +14878,58 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     )
   }
 
+  /**
+   * @description The provider instance backing a given index {@link IndexFamily}.
+   * The single place the family label maps to the concrete provider, so the
+   * scoped migration gate and any future family-aware routing agree.
+   */
+  private providerForFamily(family: IndexFamily): unknown {
+    switch (family) {
+      case 'vector':
+        return this.index
+      case 'metadata':
+        return this.metadataIndex
+      case 'graph':
+        return this.graphIndex
+    }
+  }
+
+  /**
+   * @description Whether the migration LOCK should hold for an operation that
+   * depends on the given index families:
+   * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the
+   *   conservative default for writes and unclassified reads — a write touches
+   *   every index).
+   * - `needs === []` → NEVER: a canonical-storage read consults no derived index,
+   *   so a migration elsewhere is irrelevant; it serves immediately.
+   * - `needs = [families]` → SCOPED: only those families' providers count, so a
+   *   read served from a healthy family is not blocked by an unrelated family's
+   *   one-time migration.
+   */
+  private neededFamiliesMigrating(needs?: IndexFamily[]): boolean {
+    if (needs === undefined) return this.anyProviderMigrating()
+    for (const family of needs) {
+      if (this.providerIsMigrating(this.providerForFamily(family))) return true
+    }
+    return false
+  }
+
+  /**
+   * @description The index families a {@link find} query consults, derived from
+   * its params — the read-side input to the family-scoped migration gate. A
+   * vector / near / semantic query needs `vector`; a `where` / type / subtype /
+   * service filter or an aggregate needs `metadata`; a `connected` traversal
+   * needs `graph`. A query combining shapes needs the union. Mirrors find()'s
+   * own criteria split so the gate and the executor never disagree.
+   */
+  private queryIndexFamilies(params: FindParams<T>): IndexFamily[] {
+    const needs: IndexFamily[] = []
+    if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector')
+    if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata')
+    if (params.connected) needs.push('graph')
+    return needs
+  }
+
   /**
    * @description Rich migration progress relayed verbatim from whichever provider
    * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`.
@@ -14872,11 +14958,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   /**
    * @description Block-and-queue on the coordinated migration LOCK (#18). Called
    * at the {@link ensureInitialized} choke point (and once inside `init()` before
-   * VFS bootstrap), so every data-plane read and write — and startup itself —
-   * waits here while a native provider runs its one-time 7.x → 8.0
-   * rebuild-from-canonical. The caller gets the correct answer, never a partial
-   * read of a half-built index and never a lost write. A brain that is not
-   * migrating pays a single boolean check and returns immediately.
+   * VFS bootstrap), so a data-plane operation waits here while a native provider
+   * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the
+   * correct answer, never a partial read of a half-built index and never a lost
+   * write. A brain that is not migrating pays a single boolean check and returns
+   * immediately.
+   *
+   * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the
+   * wait holds ONLY for a migration of one of those families. A read served
+   * entirely from canonical storage (`needs: []`) or from a healthy family never
+   * blocks on an unrelated family's migration. `needs` omitted = the conservative
+   * whole-brain wait (writes touch every index; startup wants all of them ready).
+   * @param needs - Index families this operation consults, or `undefined` for the
+   *   whole-brain wait.
    *
    * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The
    * migration itself is unbounded — the native provider rebuilds a billion-scale
@@ -14890,8 +14984,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * for a very large brain, runs the offline migrator. The block/release lines
    * log once per window; the flags reset on release so a later upgrade re-logs.
    */
-  private async awaitMigrationLock(): Promise<void> {
-    if (!this.anyProviderMigrating()) return // fast path — the overwhelming common case
+  private async awaitMigrationLock(needs?: IndexFamily[]): Promise<void> {
+    if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case
 
     const startedAt = Date.now()
     const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000
@@ -14904,7 +14998,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       )
     }
 
-    while (this.anyProviderMigrating()) {
+    while (this.neededFamiliesMigrating(needs)) {
       const remaining = timeoutMs - (Date.now() - startedAt)
       if (remaining <= 0) {
         const status = this.providerMigrationStatus()
@@ -15092,6 +15186,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    */
   private async ensureMetadataConsistencyProbed(): Promise<void> {
     if (this._metadataConsistencyProbed) return
+    // Defer while the metadata provider runs its one-time in-place migration:
+    // probing (and self-healing via rebuild) an index the provider is mid-rebuild
+    // would collide with the provider that owns it. Mirrors the vector deference
+    // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next
+    // read runs the probe. (The family-scoped find() gate waits on the metadata
+    // family separately before any actual filter read.)
+    if (this.providerIsMigrating(this.metadataIndex)) return
     this._metadataConsistencyProbed = true
     const provider = this.metadataIndex as {
       probeConsistency?: () => Promise<boolean>
@@ -15161,6 +15262,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
 
   async repairIndex(): Promise<void> {
     await this.ensureInitialized()
+
+    // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete
+    // defect: a delete that removed the metadata (content) leg but left the
+    // vector leg + the entity directory (a "ghost"), or left an empty directory
+    // (a "scar"). These are not live entities (getNoun needs the content leg)
+    // yet inflate enumerated counts and confuse locator resolution. Feature-
+    // detected (filesystem-only — key/prefix stores have no orphan containers);
+    // recompute counts afterward so the totals stop counting the ghosts.
+    const pruner = this.storage as {
+      pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }>
+      rebuildTypeCounts?: () => Promise<void>
+      rebuildSubtypeCounts?: () => Promise<void>
+    }
+    if (typeof pruner.pruneOrphanedEntities === 'function') {
+      const orphans = await pruner.pruneOrphanedEntities()
+      if (orphans.nouns.length + orphans.verbs.length > 0) {
+        await pruner.rebuildTypeCounts?.()
+        await pruner.rebuildSubtypeCounts?.()
+        prodLog.warn(
+          `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
+            `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
+            `partial delete, and recomputed counts.`
+        )
+      }
+    }
+
     await this.metadataIndex.detectAndRepairCorruption()
     // Lift a failed-rollback write-quarantine: force a full rebuild so the
     // derived indexes are provably reconciled with canonical, then clear the
@@ -15175,7 +15302,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         `Writes are re-enabled.`
       )
     }
-    // Cross-layer repair (ADR-004 §6): repairIndex must reconcile NATIVE derived
+    // Cross-layer repair: repairIndex must reconcile NATIVE derived
     // state from canonical, not just the JS metadata index. Consult each provider's
     // own validateInvariants() and rebuild any whose failing invariant asks for it
     // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption().