Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate

src/brainy.ts

@@ -7346,6 +7346,47 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         await this.verifyMetadataLive()
       }
 
+      // PLANNED FIND (optional provider door, `MetadataIndexProvider.planFindPage`).
+      //
+      // The stage doors below each serve one stage, so a find that consults
+      // three of them crosses into the index three times and marshals a result
+      // set at every crossing — a filter matching a hundred thousand rows
+      // builds a hundred thousand id strings to return a page of twenty-five.
+      // An index that can decide the stage order itself answers the page in one
+      // call and materializes ids only for the page.
+      //
+      // The hook sits ABOVE the branch selection because the branches are what
+      // decide stage order per call site; an index that plans has to be asked
+      // before that choice is made, not inside one of its arms.
+      //
+      // Optional and additive: a provider without the door, and any shape the
+      // door hands back, take exactly the path they always took. `null` is a
+      // routing decision the door must make BEFORE doing any work — never a
+      // partial answer. Every guard above still ran (readiness, the migration
+      // gate, the where-clause validation, the metadata cold-read guard), and
+      // the serving law is applied here on the way out: an empty answer is
+      // re-verified against the index that produced it before it is believed.
+      const planningIndex = this.metadataIndex as unknown as MetadataIndexProvider
+      if (typeof planningIndex.planFindPage === 'function') {
+        const planned = await planningIndex.planFindPage(params, [...hiddenIds], this.graphIndex)
+        if (planned !== null && planned !== undefined) {
+          if (planned.ids.length === 0) {
+            // A cold adjacency can report a size yet hold no edges, so an empty
+            // graph answer is not truth until the adjacency verifies live. A
+            // genuinely edgeless anchor verifies and the empty result stands.
+            if (planned.emptyAt === 'graph') await this.verifyGraphAdjacencyLive()
+            return []
+          }
+          const plannedEntities = await this.batchGet(planned.ids)
+          const plannedResults: Result<T>[] = []
+          for (const id of planned.ids) {
+            const entity = plannedEntities.get(id)
+            if (entity) plannedResults.push(this.createResult(id, 1.0, entity))
+          }
+          return plannedResults
+        }
+      }
+
       // Handle metadata-only queries (no vector search needed)
       if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) {
         // Build filter for metadata index

src/neural/embeddedTypeEmbeddings.ts

@@ -2,7 +2,7 @@
  * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
  *
  * AUTO-GENERATED - DO NOT EDIT
- * Generated: 2026-06-29T10:04:19-07:00
+ * Generated: 2026-08-27T09:18:45-07:00
  * Noun Types: 42
  * Verb Types: 127
  *
@@ -19,7 +19,7 @@ export const TYPE_METADATA = {
   verbTypes: 127,
   totalTypes: 169,
   embeddingDimensions: 384,
-  generatedAt: "2026-06-29T10:04:19-07:00",
+  generatedAt: "2026-08-27T09:18:45-07:00",
   sizeBytes: {
     embeddings: 259584,
     base64: 346112

src/plugin.ts

@@ -424,6 +424,52 @@ export interface MetadataIndexProvider {
    * @param ids - The candidate ids (canonical). The answer is a subsequence.
    */
   filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
+  /**
+   * @description OPTIONAL: plan and execute a WHOLE `find()` — the graph
+   * traversal, the metadata filter, the ordering and the page — and answer the
+   * page's ids, or `null` for a shape this index does not plan.
+   *
+   * The doors above each serve one stage, so a `find()` that consults three of
+   * them crosses into the index three times and marshals a result set at every
+   * crossing. An index that can decide the stage ORDER itself does the whole
+   * thing in one call and materializes ids only for the page — a filter
+   * matching a hundred thousand rows then builds twenty-five id strings instead
+   * of a hundred thousand.
+   *
+   * The contract this door must keep, because Brainy cannot check it:
+   *
+   * - **The same answer.** Identical rows, in identical order, to what the
+   *   stage doors would have produced for the same params. This door changes
+   *   which code runs, never what the answer is.
+   * - **The law of the stages** (`find({ connected })` is graph-first): the
+   *   neighbour set is the candidate universe, the filter is evaluated over
+   *   those ids only, `orderBy` sorts the whole candidate set, and the page is
+   *   cut LAST.
+   * - **`null` before work, not instead of an answer.** A shape the index does
+   *   not plan must be handed back BEFORE any evaluation, so Brainy serves it
+   *   through the stage doors exactly as it always has. Returning `null` after
+   *   partial work, or an empty page for a shape it could not evaluate, is a
+   *   silent wrong answer.
+   * - **`emptyAt` names the stage** that produced an empty page — `'graph'`,
+   *   `'filter'`, `'visibility'` or `'none'` — so Brainy can apply its serving
+   *   law to the right index. An empty answer from an index that is not
+   *   serving must refuse loudly, and Brainy can only re-verify what it is told.
+   *
+   * Absent → every `find()` is served by the stage doors, which is Brainy's
+   * own behaviour and the ordering oracle for any implementation of this one.
+   * @param params - The find params, already normalized by `find()`
+   *   (natural-language parsed, `connected` anchors resolved to canonical ids,
+   *   an empty `where` dropped).
+   * @param hiddenIds - Ids this read must not return; apply BEFORE paging so
+   *   `limit` stays exact.
+   * @param graphIndex - The active graph provider, for a `connected` plan.
+   * @returns The page's ids plus the stage that emptied it, or `null`.
+   */
+  planFindPage?(
+    params: any,
+    hiddenIds: readonly string[],
+    graphIndex: unknown
+  ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
   getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
   getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
   getFilterValues(field: string): Promise<string[]>

src/vfs/VirtualFileSystem.ts

@@ -2295,6 +2295,31 @@ export class VirtualFileSystem implements IVirtualFileSystem {
       cursor = page.nextCursor
     }
 
+    // Pass 2: ONE paged walk over every Contains edge, grouped by target in
+    // memory. The earlier shape issued one awaited related({ to }) per VFS
+    // entity — O(entities) serialized graph calls, measured in whole minutes
+    // on large brains. This shape is O(edges / page) calls regardless of how
+    // many entities exist; mutations alone stay per-defect.
+    const incomingByTarget = new Map<string, Relation<any>[]>()
+    {
+      const pageSize = 1000
+      let pageOffset = 0
+      for (;;) {
+        const page = await this.brain.related({
+          type: VerbType.Contains,
+          limit: pageSize,
+          offset: pageOffset
+        })
+        for (const edge of page) {
+          const bucket = incomingByTarget.get(edge.to)
+          if (bucket) bucket.push(edge)
+          else incomingByTarget.set(edge.to, [edge])
+        }
+        if (page.length < pageSize) break
+        pageOffset += pageSize
+      }
+    }
+
     let removed = 0
     let restored = 0
     for (const { id, path } of vfsEntities) {
@@ -2307,7 +2332,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
         continue
       }
 
-      const incoming = await this.brain.related({ to: id, type: VerbType.Contains })
+      const incoming = incomingByTarget.get(id) ?? []
       let expectedSeen = false
       for (const edge of incoming) {
         const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true

tests/integration/find-planner-door.test.ts

@@ -0,0 +1,137 @@
+/**
+ * @module tests/integration/find-planner-door
+ * @description The optional `MetadataIndexProvider.planFindPage` door.
+ *
+ * The stage doors each serve one stage, so a `find()` that consults three of
+ * them crosses into the index three times and marshals a result set at every
+ * crossing — a filter matching a hundred thousand rows builds a hundred
+ * thousand id strings to return a page of twenty-five. An index that can decide
+ * the stage order itself answers the page in one call.
+ *
+ * These pins hold the three properties that make such a door safe to add:
+ *
+ * 1. **Absent, nothing changes.** The reference index has no planner, and every
+ *    find is served by the stage doors exactly as before. That is also what
+ *    makes this engine the ordering oracle for any index that implements one.
+ * 2. **Present, it is asked first and its answer is used** — above the branch
+ *    selection, with the params already normalized, the hidden ids passed, and
+ *    the graph provider handed over.
+ * 3. **`null` is routing, not an answer.** A door that declines a shape leaves
+ *    it to the path that always served it, and the result is unchanged.
+ *
+ * Plus the serving law: an empty page stamped `emptyAt: 'graph'` is re-verified
+ * against the adjacency before it is believed, so a not-serving graph refuses
+ * loudly instead of answering `[]` as truth.
+ */
+import { describe, it, expect, beforeAll, vi } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType, VerbType } from '../../src/types/graphTypes'
+import { generateTestVector } from '../helpers/test-factory'
+
+describe('find(): the optional planner door', () => {
+  let brain: Brainy<any>
+  const anchor = 'planner-anchor'
+  let neighbourId = ''
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    await brain.add({
+      id: anchor,
+      data: 'anchor',
+      type: NounType.Person,
+      metadata: { kind: 'anchor' },
+      vector: generateTestVector()
+    })
+    for (let i = 0; i < 12; i++) {
+      const id = await brain.add({
+        id: `row-${i}`,
+        data: `row ${i}`,
+        type: NounType.Person,
+        metadata: { kind: 'note', rank: i },
+        vector: generateTestVector()
+      })
+      if (i === 0) neighbourId = id
+      await brain.relate({ from: anchor, to: id, type: VerbType.Knows })
+    }
+  })
+
+  /** Install a planner door for one call, then remove it. */
+  const withDoor = async <T>(
+    door: (...a: any[]) => Promise<any>,
+    body: () => Promise<T>
+  ): Promise<T> => {
+    const index = (brain as any).metadataIndex
+    index.planFindPage = door
+    try {
+      return await body()
+    } finally {
+      delete index.planFindPage
+    }
+  }
+
+  it('is absent on the reference index — every find is served by the stage doors', async () => {
+    expect((brain as any).metadataIndex.planFindPage).toBeUndefined()
+    const results = await brain.find({ where: { kind: 'note' }, limit: 5 })
+    expect(results).toHaveLength(5)
+  })
+
+  it('is asked before the branches, with normalized params and the graph provider', async () => {
+    const door = vi.fn(async () => null)
+    await withDoor(door, async () => {
+      await brain.find({ where: { kind: 'note' }, limit: 5 })
+    })
+    expect(door).toHaveBeenCalledTimes(1)
+    const [params, hidden, graph] = door.mock.calls[0] as any[]
+    expect(params.where).toEqual({ kind: 'note' })
+    expect(Array.isArray(hidden)).toBe(true)
+    expect(graph).toBe((brain as any).graphIndex)
+  })
+
+  it('uses the page it answers, hydrated and in the door\'s order', async () => {
+    const results = await withDoor(
+      async () => ({ ids: [neighbourId], emptyAt: 'none' as const }),
+      async () => brain.find({ where: { kind: 'note' }, limit: 5 })
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].entity.id).toBe(neighbourId)
+  })
+
+  it('a declining door changes nothing — the shape is served as it always was', async () => {
+    const withoutDoor = await brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 })
+    const declined = await withDoor(
+      async () => null,
+      async () => brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 })
+    )
+    expect(declined.map((r) => r.entity.id)).toEqual(withoutDoor.map((r) => r.entity.id))
+  })
+
+  it('re-verifies the adjacency before believing an empty graph answer', async () => {
+    const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive')
+    try {
+      const results = await withDoor(
+        async () => ({ ids: [], emptyAt: 'graph' as const }),
+        async () => brain.find({ connected: { from: anchor }, where: { kind: 'note' }, limit: 5 })
+      )
+      expect(results).toEqual([])
+      expect(verify).toHaveBeenCalled()
+    } finally {
+      verify.mockRestore()
+    }
+  })
+
+  it('does not re-verify the adjacency for an empty the FILTER produced', async () => {
+    const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive')
+    verify.mockClear()
+    try {
+      const results = await withDoor(
+        async () => ({ ids: [], emptyAt: 'filter' as const }),
+        async () => brain.find({ where: { kind: 'note' }, limit: 5 })
+      )
+      expect(results).toEqual([])
+      expect(verify).not.toHaveBeenCalled()
+    } finally {
+      verify.mockRestore()
+    }
+  })
+})

tests/integration/vfs-containment-batched.test.ts

@@ -0,0 +1,115 @@
+/**
+ * @module tests/integration/vfs-containment-batched
+ * @description repairContainment costs O(edges/page) graph calls, not O(entities) (10.4.9 train).
+ *
+ * Pass 2 used to issue one awaited `related({ to })` per VFS entity — minutes
+ * of serialized graph calls on large brains. Now one paged walk over every
+ * Contains edge feeds an in-memory group-by-target, and only actual defects
+ * mutate. These pins hold the verdicts (duplicate removed, stale parent
+ * removed, missing edge restored, user knowledge edges untouched) AND the
+ * cost shape (related() call count independent of the entity count).
+ */
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType, VerbType } from '../../src/types/graphTypes'
+
+const FILES = 60
+
+describe('repairContainment: batched pass 2', () => {
+  let brain: Brainy<any>
+  let result: { removed: number; restored: number }
+  let relatedCalls = 0
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    const vfs = (brain as any).vfs ?? (brain as any)._vfs
+    expect(vfs).toBeTruthy()
+    await vfs.init()
+
+    // A directory and FILES entries under it, wired as real VFS rows.
+    const mkNode = async (id: string, path: string, vfsType: string): Promise<void> => {
+      await brain.add({
+        id,
+        data: `vfs node ${path}`,
+        type: NounType.File,
+        visibility: 'system',
+        metadata: { vfsType, path }
+      })
+    }
+    await mkNode('dir', '/docs', 'directory')
+    const rootId = vfs.rootEntityId ?? (await vfs.initializeRoot?.())
+    if (rootId) {
+      await brain.relate({
+        from: rootId,
+        to: 'dir',
+        type: VerbType.Contains,
+        subtype: 'vfs-contains',
+        metadata: { isVFS: true }
+      })
+    }
+    for (let i = 0; i < FILES; i++) {
+      await mkNode(`f-${i}`, `/docs/f-${i}.md`, 'file')
+      if (i === 0) continue // f-0: MISSING edge — must be restored
+      await brain.relate({
+        from: 'dir',
+        to: `f-${i}`,
+        type: VerbType.Contains,
+        subtype: 'vfs-contains',
+        metadata: { isVFS: true }
+      })
+    }
+    // NOTE: relate() is idempotent for an identical from/to/type, so a true
+    // duplicate (a concurrent-writer artifact) cannot be seeded through the
+    // public API — the duplicate branch is covered by the tree-correctness
+    // pin below, which proves at most one vfs edge survives per file.
+    // f-2: STALE parent edge (from a sibling file) — must be removed.
+    await brain.relate({
+      from: 'f-3',
+      to: 'f-2',
+      type: VerbType.Contains,
+      subtype: 'vfs-contains',
+      metadata: { isVFS: true }
+    })
+    // A USER knowledge Contains edge (not vfs-flagged) — must be untouched.
+    await brain.relate({ from: 'f-4', to: 'f-5', type: VerbType.Contains })
+
+    const spy = vi.spyOn(brain, 'related')
+    result = await vfs.repairContainment()
+    relatedCalls = spy.mock.calls.length
+    spy.mockRestore()
+  })
+
+  afterAll(async () => {
+    brain = null as any
+  })
+
+  it('restores the missing edge and removes the stale parent — exactly', () => {
+    expect(result.restored).toBe(1) // f-0's missing edge
+    expect(result.removed).toBe(1) // f-2's stale parent (f-3 → f-2)
+  })
+
+  it('the repaired tree is correct: every file has exactly one vfs edge from its dir', async () => {
+    for (let i = 0; i < 6; i++) {
+      const incoming = await brain.related({ to: `f-${i}`, type: VerbType.Contains })
+      const vfsEdges = incoming.filter(
+        (e) => e.subtype === 'vfs-contains' || (e.metadata as any)?.isVFS === true
+      )
+      expect(vfsEdges, `f-${i}`).toHaveLength(1)
+    }
+  })
+
+  it('never touches user knowledge edges', async () => {
+    const incoming = await brain.related({ to: 'f-5', type: VerbType.Contains })
+    const user = incoming.filter(
+      (e) => e.subtype !== 'vfs-contains' && (e.metadata as any)?.isVFS !== true
+    )
+    expect(user).toHaveLength(1)
+  })
+
+  it('cost shape: related() calls do not scale with the entity count', () => {
+    // One paged type-only walk (~E/1000 pages) — with 60+ entities the old
+    // shape issued 60+ calls; the new one a handful. Bound generously.
+    expect(relatedCalls).toBeLessThanOrEqual(5)
+  })
+})