Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate

src/brainy.ts

@@ -7346,6 +7346,47 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         await this.verifyMetadataLive()
       }
 
+      // PLANNED FIND (optional provider door, `MetadataIndexProvider.planFindPage`).
+      //
+      // The stage doors below each serve one stage, so a find that consults
+      // three of them crosses into the index three times and marshals a result
+      // set at every crossing — a filter matching a hundred thousand rows
+      // builds a hundred thousand id strings to return a page of twenty-five.
+      // An index that can decide the stage order itself answers the page in one
+      // call and materializes ids only for the page.
+      //
+      // The hook sits ABOVE the branch selection because the branches are what
+      // decide stage order per call site; an index that plans has to be asked
+      // before that choice is made, not inside one of its arms.
+      //
+      // Optional and additive: a provider without the door, and any shape the
+      // door hands back, take exactly the path they always took. `null` is a
+      // routing decision the door must make BEFORE doing any work — never a
+      // partial answer. Every guard above still ran (readiness, the migration
+      // gate, the where-clause validation, the metadata cold-read guard), and
+      // the serving law is applied here on the way out: an empty answer is
+      // re-verified against the index that produced it before it is believed.
+      const planningIndex = this.metadataIndex as unknown as MetadataIndexProvider
+      if (typeof planningIndex.planFindPage === 'function') {
+        const planned = await planningIndex.planFindPage(params, [...hiddenIds], this.graphIndex)
+        if (planned !== null && planned !== undefined) {
+          if (planned.ids.length === 0) {
+            // A cold adjacency can report a size yet hold no edges, so an empty
+            // graph answer is not truth until the adjacency verifies live. A
+            // genuinely edgeless anchor verifies and the empty result stands.
+            if (planned.emptyAt === 'graph') await this.verifyGraphAdjacencyLive()
+            return []
+          }
+          const plannedEntities = await this.batchGet(planned.ids)
+          const plannedResults: Result<T>[] = []
+          for (const id of planned.ids) {
+            const entity = plannedEntities.get(id)
+            if (entity) plannedResults.push(this.createResult(id, 1.0, entity))
+          }
+          return plannedResults
+        }
+      }
+
       // Handle metadata-only queries (no vector search needed)
       if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) {
         // Build filter for metadata index

src/neural/embeddedTypeEmbeddings.ts

@@ -2,7 +2,7 @@
  * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
  *
  * AUTO-GENERATED - DO NOT EDIT
- * Generated: 2026-06-29T10:04:19-07:00
+ * Generated: 2026-08-27T09:18:45-07:00
  * Noun Types: 42
  * Verb Types: 127
  *
@@ -19,7 +19,7 @@ export const TYPE_METADATA = {
   verbTypes: 127,
   totalTypes: 169,
   embeddingDimensions: 384,
-  generatedAt: "2026-06-29T10:04:19-07:00",
+  generatedAt: "2026-08-27T09:18:45-07:00",
   sizeBytes: {
     embeddings: 259584,
     base64: 346112

src/plugin.ts

@@ -424,6 +424,52 @@ export interface MetadataIndexProvider {
    * @param ids - The candidate ids (canonical). The answer is a subsequence.
    */
   filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
+  /**
+   * @description OPTIONAL: plan and execute a WHOLE `find()` — the graph
+   * traversal, the metadata filter, the ordering and the page — and answer the
+   * page's ids, or `null` for a shape this index does not plan.
+   *
+   * The doors above each serve one stage, so a `find()` that consults three of
+   * them crosses into the index three times and marshals a result set at every
+   * crossing. An index that can decide the stage ORDER itself does the whole
+   * thing in one call and materializes ids only for the page — a filter
+   * matching a hundred thousand rows then builds twenty-five id strings instead
+   * of a hundred thousand.
+   *
+   * The contract this door must keep, because Brainy cannot check it:
+   *
+   * - **The same answer.** Identical rows, in identical order, to what the
+   *   stage doors would have produced for the same params. This door changes
+   *   which code runs, never what the answer is.
+   * - **The law of the stages** (`find({ connected })` is graph-first): the
+   *   neighbour set is the candidate universe, the filter is evaluated over
+   *   those ids only, `orderBy` sorts the whole candidate set, and the page is
+   *   cut LAST.
+   * - **`null` before work, not instead of an answer.** A shape the index does
+   *   not plan must be handed back BEFORE any evaluation, so Brainy serves it
+   *   through the stage doors exactly as it always has. Returning `null` after
+   *   partial work, or an empty page for a shape it could not evaluate, is a
+   *   silent wrong answer.
+   * - **`emptyAt` names the stage** that produced an empty page — `'graph'`,
+   *   `'filter'`, `'visibility'` or `'none'` — so Brainy can apply its serving
+   *   law to the right index. An empty answer from an index that is not
+   *   serving must refuse loudly, and Brainy can only re-verify what it is told.
+   *
+   * Absent → every `find()` is served by the stage doors, which is Brainy's
+   * own behaviour and the ordering oracle for any implementation of this one.
+   * @param params - The find params, already normalized by `find()`
+   *   (natural-language parsed, `connected` anchors resolved to canonical ids,
+   *   an empty `where` dropped).
+   * @param hiddenIds - Ids this read must not return; apply BEFORE paging so
+   *   `limit` stays exact.
+   * @param graphIndex - The active graph provider, for a `connected` plan.
+   * @returns The page's ids plus the stage that emptied it, or `null`.
+   */
+  planFindPage?(
+    params: any,
+    hiddenIds: readonly string[],
+    graphIndex: unknown
+  ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
   getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
   getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
   getFilterValues(field: string): Promise<string[]>

src/vfs/VirtualFileSystem.ts

@@ -2295,6 +2295,31 @@ export class VirtualFileSystem implements IVirtualFileSystem {
       cursor = page.nextCursor
     }
 
+    // Pass 2: ONE paged walk over every Contains edge, grouped by target in
+    // memory. The earlier shape issued one awaited related({ to }) per VFS
+    // entity — O(entities) serialized graph calls, measured in whole minutes
+    // on large brains. This shape is O(edges / page) calls regardless of how
+    // many entities exist; mutations alone stay per-defect.
+    const incomingByTarget = new Map<string, Relation<any>[]>()
+    {
+      const pageSize = 1000
+      let pageOffset = 0
+      for (;;) {
+        const page = await this.brain.related({
+          type: VerbType.Contains,
+          limit: pageSize,
+          offset: pageOffset
+        })
+        for (const edge of page) {
+          const bucket = incomingByTarget.get(edge.to)
+          if (bucket) bucket.push(edge)
+          else incomingByTarget.set(edge.to, [edge])
+        }
+        if (page.length < pageSize) break
+        pageOffset += pageSize
+      }
+    }
+
     let removed = 0
     let restored = 0
     for (const { id, path } of vfsEntities) {
@@ -2307,7 +2332,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
         continue
       }
 
-      const incoming = await this.brain.related({ to: id, type: VerbType.Contains })
+      const incoming = incomingByTarget.get(id) ?? []
       let expectedSeen = false
       for (const edge of incoming) {
         const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true