Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate

tests/integration/find-planner-door.test.ts

@@ -0,0 +1,137 @@
+/**
+ * @module tests/integration/find-planner-door
+ * @description The optional `MetadataIndexProvider.planFindPage` door.
+ *
+ * The stage doors each serve one stage, so a `find()` that consults three of
+ * them crosses into the index three times and marshals a result set at every
+ * crossing — a filter matching a hundred thousand rows builds a hundred
+ * thousand id strings to return a page of twenty-five. An index that can decide
+ * the stage order itself answers the page in one call.
+ *
+ * These pins hold the three properties that make such a door safe to add:
+ *
+ * 1. **Absent, nothing changes.** The reference index has no planner, and every
+ *    find is served by the stage doors exactly as before. That is also what
+ *    makes this engine the ordering oracle for any index that implements one.
+ * 2. **Present, it is asked first and its answer is used** — above the branch
+ *    selection, with the params already normalized, the hidden ids passed, and
+ *    the graph provider handed over.
+ * 3. **`null` is routing, not an answer.** A door that declines a shape leaves
+ *    it to the path that always served it, and the result is unchanged.
+ *
+ * Plus the serving law: an empty page stamped `emptyAt: 'graph'` is re-verified
+ * against the adjacency before it is believed, so a not-serving graph refuses
+ * loudly instead of answering `[]` as truth.
+ */
+import { describe, it, expect, beforeAll, vi } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType, VerbType } from '../../src/types/graphTypes'
+import { generateTestVector } from '../helpers/test-factory'
+
+describe('find(): the optional planner door', () => {
+  let brain: Brainy<any>
+  const anchor = 'planner-anchor'
+  let neighbourId = ''
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    await brain.add({
+      id: anchor,
+      data: 'anchor',
+      type: NounType.Person,
+      metadata: { kind: 'anchor' },
+      vector: generateTestVector()
+    })
+    for (let i = 0; i < 12; i++) {
+      const id = await brain.add({
+        id: `row-${i}`,
+        data: `row ${i}`,
+        type: NounType.Person,
+        metadata: { kind: 'note', rank: i },
+        vector: generateTestVector()
+      })
+      if (i === 0) neighbourId = id
+      await brain.relate({ from: anchor, to: id, type: VerbType.Knows })
+    }
+  })
+
+  /** Install a planner door for one call, then remove it. */
+  const withDoor = async <T>(
+    door: (...a: any[]) => Promise<any>,
+    body: () => Promise<T>
+  ): Promise<T> => {
+    const index = (brain as any).metadataIndex
+    index.planFindPage = door
+    try {
+      return await body()
+    } finally {
+      delete index.planFindPage
+    }
+  }
+
+  it('is absent on the reference index — every find is served by the stage doors', async () => {
+    expect((brain as any).metadataIndex.planFindPage).toBeUndefined()
+    const results = await brain.find({ where: { kind: 'note' }, limit: 5 })
+    expect(results).toHaveLength(5)
+  })
+
+  it('is asked before the branches, with normalized params and the graph provider', async () => {
+    const door = vi.fn(async () => null)
+    await withDoor(door, async () => {
+      await brain.find({ where: { kind: 'note' }, limit: 5 })
+    })
+    expect(door).toHaveBeenCalledTimes(1)
+    const [params, hidden, graph] = door.mock.calls[0] as any[]
+    expect(params.where).toEqual({ kind: 'note' })
+    expect(Array.isArray(hidden)).toBe(true)
+    expect(graph).toBe((brain as any).graphIndex)
+  })
+
+  it('uses the page it answers, hydrated and in the door\'s order', async () => {
+    const results = await withDoor(
+      async () => ({ ids: [neighbourId], emptyAt: 'none' as const }),
+      async () => brain.find({ where: { kind: 'note' }, limit: 5 })
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].entity.id).toBe(neighbourId)
+  })
+
+  it('a declining door changes nothing — the shape is served as it always was', async () => {
+    const withoutDoor = await brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 })
+    const declined = await withDoor(
+      async () => null,
+      async () => brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 })
+    )
+    expect(declined.map((r) => r.entity.id)).toEqual(withoutDoor.map((r) => r.entity.id))
+  })
+
+  it('re-verifies the adjacency before believing an empty graph answer', async () => {
+    const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive')
+    try {
+      const results = await withDoor(
+        async () => ({ ids: [], emptyAt: 'graph' as const }),
+        async () => brain.find({ connected: { from: anchor }, where: { kind: 'note' }, limit: 5 })
+      )
+      expect(results).toEqual([])
+      expect(verify).toHaveBeenCalled()
+    } finally {
+      verify.mockRestore()
+    }
+  })
+
+  it('does not re-verify the adjacency for an empty the FILTER produced', async () => {
+    const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive')
+    verify.mockClear()
+    try {
+      const results = await withDoor(
+        async () => ({ ids: [], emptyAt: 'filter' as const }),
+        async () => brain.find({ where: { kind: 'note' }, limit: 5 })
+      )
+      expect(results).toEqual([])
+      expect(verify).not.toHaveBeenCalled()
+    } finally {
+      verify.mockRestore()
+    }
+  })
+})

tests/integration/vfs-containment-batched.test.ts

@@ -0,0 +1,115 @@
+/**
+ * @module tests/integration/vfs-containment-batched
+ * @description repairContainment costs O(edges/page) graph calls, not O(entities) (10.4.9 train).
+ *
+ * Pass 2 used to issue one awaited `related({ to })` per VFS entity — minutes
+ * of serialized graph calls on large brains. Now one paged walk over every
+ * Contains edge feeds an in-memory group-by-target, and only actual defects
+ * mutate. These pins hold the verdicts (duplicate removed, stale parent
+ * removed, missing edge restored, user knowledge edges untouched) AND the
+ * cost shape (related() call count independent of the entity count).
+ */
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType, VerbType } from '../../src/types/graphTypes'
+
+const FILES = 60
+
+describe('repairContainment: batched pass 2', () => {
+  let brain: Brainy<any>
+  let result: { removed: number; restored: number }
+  let relatedCalls = 0
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    const vfs = (brain as any).vfs ?? (brain as any)._vfs
+    expect(vfs).toBeTruthy()
+    await vfs.init()
+
+    // A directory and FILES entries under it, wired as real VFS rows.
+    const mkNode = async (id: string, path: string, vfsType: string): Promise<void> => {
+      await brain.add({
+        id,
+        data: `vfs node ${path}`,
+        type: NounType.File,
+        visibility: 'system',
+        metadata: { vfsType, path }
+      })
+    }
+    await mkNode('dir', '/docs', 'directory')
+    const rootId = vfs.rootEntityId ?? (await vfs.initializeRoot?.())
+    if (rootId) {
+      await brain.relate({
+        from: rootId,
+        to: 'dir',
+        type: VerbType.Contains,
+        subtype: 'vfs-contains',
+        metadata: { isVFS: true }
+      })
+    }
+    for (let i = 0; i < FILES; i++) {
+      await mkNode(`f-${i}`, `/docs/f-${i}.md`, 'file')
+      if (i === 0) continue // f-0: MISSING edge — must be restored
+      await brain.relate({
+        from: 'dir',
+        to: `f-${i}`,
+        type: VerbType.Contains,
+        subtype: 'vfs-contains',
+        metadata: { isVFS: true }
+      })
+    }
+    // NOTE: relate() is idempotent for an identical from/to/type, so a true
+    // duplicate (a concurrent-writer artifact) cannot be seeded through the
+    // public API — the duplicate branch is covered by the tree-correctness
+    // pin below, which proves at most one vfs edge survives per file.
+    // f-2: STALE parent edge (from a sibling file) — must be removed.
+    await brain.relate({
+      from: 'f-3',
+      to: 'f-2',
+      type: VerbType.Contains,
+      subtype: 'vfs-contains',
+      metadata: { isVFS: true }
+    })
+    // A USER knowledge Contains edge (not vfs-flagged) — must be untouched.
+    await brain.relate({ from: 'f-4', to: 'f-5', type: VerbType.Contains })
+
+    const spy = vi.spyOn(brain, 'related')
+    result = await vfs.repairContainment()
+    relatedCalls = spy.mock.calls.length
+    spy.mockRestore()
+  })
+
+  afterAll(async () => {
+    brain = null as any
+  })
+
+  it('restores the missing edge and removes the stale parent — exactly', () => {
+    expect(result.restored).toBe(1) // f-0's missing edge
+    expect(result.removed).toBe(1) // f-2's stale parent (f-3 → f-2)
+  })
+
+  it('the repaired tree is correct: every file has exactly one vfs edge from its dir', async () => {
+    for (let i = 0; i < 6; i++) {
+      const incoming = await brain.related({ to: `f-${i}`, type: VerbType.Contains })
+      const vfsEdges = incoming.filter(
+        (e) => e.subtype === 'vfs-contains' || (e.metadata as any)?.isVFS === true
+      )
+      expect(vfsEdges, `f-${i}`).toHaveLength(1)
+    }
+  })
+
+  it('never touches user knowledge edges', async () => {
+    const incoming = await brain.related({ to: 'f-5', type: VerbType.Contains })
+    const user = incoming.filter(
+      (e) => e.subtype !== 'vfs-contains' && (e.metadata as any)?.isVFS !== true
+    )
+    expect(user).toHaveLength(1)
+  })
+
+  it('cost shape: related() calls do not scale with the entity count', () => {
+    // One paged type-only walk (~E/1000 pages) — with 60+ entities the old
+    // shape issued 60+ calls; the new one a handful. Bound generously.
+    expect(relatedCalls).toBeLessThanOrEqual(5)
+  })
+})