fix(find): connected finds are graph-first — neighbours, then the filter over those ids, then the page

With `connected` present, find() materialized the whole-store filtered id
list, paged it, hydrated the page, and only then intersected with the
neighbour set. Every such call paid O(store) for the filter and the
hydration of rows that were never neighbours, and a neighbour outside the
first page of the filtered STORE was silently dropped — the answer depended
on the store's order and the page size.

The neighbour set is now the candidate universe: resolved first from the
adjacency, the metadata filter evaluated over those ids only through the
provider's own evaluation (a new optional `filterIdsWithin` door on
MetadataIndexProvider; the reference index implements it from its own
getIdsForFilter so the two can never disagree; a provider without it is
served by the whole-store answer intersected here), `orderBy` sorts the
whole neighbour set before the page is cut, and the vector leg walks the
neighbours as its candidate set. The text leg of a hybrid find keeps its
post-intersection — it has no candidate door.

Pinned in tests/integration/find-connected-order.test.ts: paging reaches
every matching neighbour and never a non-neighbour; a `missing` negation is
evaluated over the neighbours; the index is asked about the neighbour ids
only and hydration is one page; orderBy sorts the whole set; the vector leg
stays inside the neighbours; an edgeless anchor answers [] before the
filter is asked.

src/brainy.ts

@@ -7470,7 +7470,37 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       // JS path — there the materialized `candidateIds` restricts the walk instead.
       let preResolvedAllowedIds: OpaqueIdSet | undefined
 
-      if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
+      // Graph-first law (10.4.8, BRAINY-PROD-LATENCY-TRIAD rounds 44/45): with
+      // `connected` present the NEIGHBOUR SET is the candidate universe. It is
+      // resolved first from the adjacency (O(neighbours)), the metadata filter
+      // is evaluated over those ids only, and paging happens LAST. The earlier
+      // order materialized the whole-store filtered id list, paged it, hydrated
+      // the page, and only then intersected with the neighbours — O(store) per
+      // call, and a neighbour outside the first page was silently dropped.
+      let graphFirstIds: string[] | null = null
+      if (hasGraphCriteria) {
+        graphFirstIds = await this.resolveConnectedIds(params)
+        if (hiddenIds.size > 0) {
+          graphFirstIds = graphFirstIds.filter((id) => !hiddenIds.has(id))
+        }
+        if (
+          graphFirstIds.length > 0 &&
+          (params.where || params.type || params.subtype || params.service || params.excludeVFS)
+        ) {
+          preResolvedFilter = this.buildMetadataFilter(params)
+          graphFirstIds = await this.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
+        }
+        if (graphFirstIds.length === 0) {
+          return []
+        }
+        if (!hasVectorSearchCriteria) {
+          return await this.pageConnectedIds(params, graphFirstIds)
+        }
+        // The vector leg walks ONLY the neighbours (its candidate walk). The
+        // filter is already applied above, so no opaque universe is produced —
+        // it would describe the whole store, not the neighbour set.
+        preResolvedMetadataIds = graphFirstIds
+      } else if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
         preResolvedFilter = this.buildMetadataFilter(params)
         preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)
 
@@ -7659,9 +7689,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         }
       }
 
-      // Graph search component with O(1) traversal
+      // The text leg of a hybrid find has no candidate door, so its hits are
-      if (params.connected) {
+      // held to the neighbour set here; the vector leg walked only the neighbours.
-        results = await this.executeGraphSearch(params, results)
+      if (graphFirstIds !== null && results.length > 0) {
+        const neighbourSet = new Set(graphFirstIds)
+        results = results.filter((r) => neighbourSet.has(r.id))
       }
 
       // Apply fusion scoring if requested
@@ -12776,6 +12808,29 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
   }
 
+  /**
+   * The id-scoped twin of {@link filterIdsBelted}: evaluate `filter` over `ids`
+   * only, through the provider's own evaluation so the answer can never drift
+   * from `getIdsForFilter`'s. A provider without the door is served by its
+   * whole-store answer intersected here (the reference index implements the
+   * door itself). Same belt: field refusals cross as `BrainyFieldRefusal`.
+   */
+  private async filterIdsWithinBelted(filter: unknown, ids: readonly string[]): Promise<string[]> {
+    this.ensureIndexesLoaded(['metadata'])
+    const mip = this.metadataIndex as unknown as MetadataIndexProvider
+    try {
+      if (typeof mip.filterIdsWithin === 'function') {
+        return await mip.filterIdsWithin(filter, ids)
+      }
+      const matched = new Set(await this.metadataIndex.getIdsForFilter(filter))
+      return ids.filter((id) => matched.has(id))
+    } catch (err) {
+      const normalized = asBrainyFieldRefusal(err)
+      if (normalized) throw normalized
+      throw err
+    }
+  }
+
   async getIndexStatus(): Promise<{
     initialized: boolean
     /** `true` once open()'s index-build-if-needed step has run. Named for API
@@ -15759,16 +15814,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   }
   
   /**
-   * Execute graph search component.
+   * Resolve `params.connected` to the neighbour id set — the graph-first
+   * find's candidate universe (deterministic traversal order, anchors excluded).
    *
    * Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via
-   * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read
+   * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. An empty set
-   * only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via`
+   * is re-verified against the adjacency before it is believed — a not-serving
-   * were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the
+   * adjacency throws rather than answering `[]` as truth.
-   * immediate neighbour at every depth.
    */
-  private async executeGraphSearch(params: FindParams<T>, existingResults: Result<T>[]): Promise<Result<T>[]> {
+  private async resolveConnectedIds(params: FindParams<T>): Promise<string[]> {
-    if (!params.connected) return existingResults
+    if (!params.connected) return []
 
     const { from, to, depth, direction = 'both' } = params.connected
     const via = params.connected.via ?? params.connected.type
@@ -15822,8 +15877,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           if (anchorInt === undefined) return new Set() // unmapped → no relations
 
           const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
-          // No limit: match the JS BFS exactly — overall result limiting happens
+          // No limit: match the JS BFS exactly — the page is cut downstream,
-          // downstream against existingResults.
+          // after the metadata filter, by pageConnectedIds / the candidate walk.
           const reachedInts = await provider.findConnectedSubtype(
             anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
           )
@@ -15908,22 +15963,44 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       await this.verifyGraphAdjacencyLive()
     }
 
-    // Filter existing results to only connected entities
+    return [...connectedIds]
-    if (existingResults.length > 0) {
-      return existingResults.filter(r => connectedIds.has(r.id))
   }
 
-    // Batch-load connected entities for fast cloud-storage performance
+  /**
+   * Page and hydrate an already-filtered neighbour set — the pure graph (and
+   * graph + metadata) find's tail. `orderBy` sorts the WHOLE set by field value
+   * before the page is cut (never the page after), null values last on `asc`
+   * and first on `desc`; without `orderBy` the traversal order stands.
+   */
+  private async pageConnectedIds(params: FindParams<T>, ids: string[]): Promise<Result<T>[]> {
+    const limit = params.limit || 10
+    const offset = params.offset || 0
+    let ordered = ids
+    if (params.orderBy) {
+      const field = params.orderBy
+      const asc = (params.order || 'asc') === 'asc'
+      const valued = await Promise.all(
+        ids.map(async (id) => ({ id, value: await this.metadataIndex.getFieldValueForEntity(id, field) }))
+      )
+      valued.sort((a, b) => {
+        if (a.value == null && b.value == null) return 0
+        if (a.value == null) return asc ? 1 : -1
+        if (b.value == null) return asc ? -1 : 1
+        if (a.value === b.value) return 0
+        const comparison = a.value < b.value ? -1 : 1
+        return asc ? comparison : -comparison
+      })
+      ordered = valued.map((v) => v.id)
+    }
+    const pageIds = ordered.slice(offset, offset + limit)
+    const entitiesMap = await this.batchGet(pageIds)
     const results: Result<T>[] = []
-    const ids = [...connectedIds]
+    for (const id of pageIds) {
-    const entitiesMap = await this.batchGet(ids)
-    for (const id of ids) {
       const entity = entitiesMap.get(id)
       if (entity) {
         results.push(this.createResult(id, 1.0, entity))
       }
     }
-
     return results
   }
   

src/plugin.ts

@@ -411,6 +411,19 @@ export interface MetadataIndexProvider {
    * @returns The matching id universe as an opaque set.
    */
   getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
+  /**
+   * @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the
+   * survivors in the caller's order — the door a graph-first
+   * `find({ connected, where })` walks. The neighbour set is the universe there,
+   * so the filter must cost O(|ids|) membership checks, never a whole-store
+   * materialization. A native index answers from its roaring filter result
+   * (membership by entity int); the reference index answers from its own
+   * `getIdsForFilter`, so the two doors can never disagree. Absent → Brainy
+   * intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)).
+   * @param filter - The same filter shape accepted by `getIdsForFilter`.
+   * @param ids - The candidate ids (canonical). The answer is a subsequence.
+   */
+  filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
   getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
   getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
   getFilterValues(field: string): Promise<string[]>

src/utils/metadataIndex.ts

@@ -2575,6 +2575,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
   /** Once-per-field flag for the fallback-degradation announcement. */
   private static announcedFallbackSorts = new Set<string>()
 
+  /**
+   * Evaluate `filter` over `ids` only — the graph-first find's door (the
+   * neighbour set filtered by id, never the store filtered and then
+   * intersected). This index answers from its own `getIdsForFilter`, so the
+   * two doors cannot disagree; the cost is that of the filter over this
+   * in-memory index, and the answer keeps the caller's order.
+   */
+  async filterIdsWithin(filter: any, ids: readonly string[]): Promise<string[]> {
+    if (ids.length === 0) return []
+    const matched = new Set(await this.getIdsForFilter(filter))
+    return ids.filter((id) => matched.has(id))
+  }
+
   async getSortedIdsForFilter(
     filter: any,
     orderBy: string,

tests/integration/find-connected-order.test.ts

@@ -0,0 +1,165 @@
+/**
+ * @module tests/integration/find-connected-order
+ * @description The graph-first law for `find({ connected })` (10.4.8).
+ *
+ * With `connected` present the neighbour set is the candidate universe: it is
+ * resolved from the adjacency first, the metadata filter is evaluated over
+ * those ids only, and the page is cut last. The earlier order materialized the
+ * whole-store filtered id list, paged it, hydrated the page, and only then
+ * intersected with the neighbours — so a neighbour outside the first page of
+ * the filtered STORE was silently dropped, and every call paid O(store).
+ *
+ * These pins hold both halves. The answer: every matching neighbour is
+ * reachable by paging, a non-neighbour never appears, a negation (`missing`)
+ * is evaluated over the neighbours, `orderBy` sorts the whole neighbour set
+ * before the page is cut, and the vector leg walks the neighbours only. The
+ * cost shape: the metadata index is asked about the neighbour ids only, and
+ * hydration is one page — never the store.
+ */
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType, VerbType } from '../../src/types/graphTypes'
+import { v5 } from '../../src/universal/uuid'
+import { generateTestVector } from '../helpers/test-factory'
+
+/** Matching rows that are NOT neighbours — added FIRST, so the whole-store filtered list leads with them. */
+const NOISE = 120
+/** Matching rows that ARE neighbours of the anchor. */
+const NEIGHBOURS = 30
+/** Neighbours carrying `retracted: true` — excluded by the `missing` negation. */
+const RETRACTED = 4
+
+describe('find({ connected }) is graph-first: neighbours → filter → page', () => {
+  let brain: Brainy<any>
+  const anchor = 'anchor'
+  const sharedVector = generateTestVector()
+  const neighbourIds = new Set(Array.from({ length: NEIGHBOURS }, (_, i) => v5(`nb-${i}`)))
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    await brain.add({
+      id: anchor,
+      data: 'the anchor',
+      type: NounType.Person,
+      metadata: { kind: 'anchor' },
+      vector: generateTestVector()
+    })
+    for (let i = 0; i < NOISE; i++) {
+      await brain.add({
+        id: `noise-${i}`,
+        data: `noise ${i}`,
+        type: NounType.Person,
+        metadata: { kind: 'note', rank: 1000 + i },
+        vector: sharedVector
+      })
+    }
+    for (let i = 0; i < NEIGHBOURS; i++) {
+      await brain.add({
+        id: `nb-${i}`,
+        data: `neighbour ${i}`,
+        type: NounType.Person,
+        metadata: { kind: 'note', rank: i + 1, ...(i < RETRACTED ? { retracted: true } : {}) },
+        vector: sharedVector
+      })
+      await brain.relate({ from: anchor, to: `nb-${i}`, type: VerbType.Knows })
+    }
+  })
+
+  afterAll(async () => {
+    brain = null as any
+  })
+
+  it('returns the matching neighbours page by page — none dropped, never a non-neighbour', async () => {
+    const seen = new Set<string>()
+    for (let offset = 0; offset <= NEIGHBOURS; offset += 10) {
+      const page = await brain.find({
+        connected: { from: anchor, direction: 'out' },
+        where: { kind: 'note' },
+        limit: 10,
+        offset
+      })
+      expect(page).toHaveLength(offset < NEIGHBOURS ? 10 : 0)
+      for (const r of page) {
+        expect(neighbourIds.has(r.entity.id)).toBe(true)
+        expect(seen.has(r.entity.id)).toBe(false)
+        seen.add(r.entity.id)
+      }
+    }
+    expect(seen.size).toBe(NEIGHBOURS)
+  })
+
+  it('evaluates a negation (`missing`) over the neighbour set, not the store', async () => {
+    const results = await brain.find({
+      connected: { from: anchor, direction: 'out' },
+      where: { kind: 'note', retracted: { missing: true } },
+      limit: 100
+    })
+    expect(results).toHaveLength(NEIGHBOURS - RETRACTED)
+    for (const r of results) {
+      expect(neighbourIds.has(r.entity.id)).toBe(true)
+      expect(r.entity.metadata.retracted).toBeUndefined()
+    }
+  })
+
+  it('asks the metadata index about the neighbour ids only, and hydrates one page', async () => {
+    const index = (brain as any).metadataIndex
+    const within = vi.spyOn(index, 'filterIdsWithin')
+    const hydrate = vi.spyOn(brain as any, 'batchGet')
+    try {
+      const results = await brain.find({
+        connected: { from: anchor, direction: 'out' },
+        where: { kind: 'note' },
+        limit: 10
+      })
+      expect(results).toHaveLength(10)
+      expect(within).toHaveBeenCalledTimes(1)
+      const askedIds = within.mock.calls[0][1] as string[]
+      expect(askedIds).toHaveLength(NEIGHBOURS)
+      for (const id of askedIds) expect(neighbourIds.has(id)).toBe(true)
+      expect(hydrate).toHaveBeenCalledTimes(1)
+      expect(hydrate.mock.calls[0][0]).toHaveLength(10)
+    } finally {
+      within.mockRestore()
+      hydrate.mockRestore()
+    }
+  })
+
+  it('orders the WHOLE neighbour set before cutting the page', async () => {
+    const results = await brain.find({
+      connected: { from: anchor, direction: 'out' },
+      where: { kind: 'note' },
+      orderBy: 'rank',
+      order: 'desc',
+      limit: 5
+    })
+    expect(results.map((r) => r.entity.metadata.rank)).toEqual([30, 29, 28, 27, 26])
+  })
+
+  it('walks the vector leg over the neighbours only', async () => {
+    const results = await brain.find({
+      vector: sharedVector,
+      connected: { from: anchor, direction: 'out' },
+      where: { kind: 'note' },
+      limit: 5
+    })
+    expect(results).toHaveLength(5)
+    for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true)
+  })
+
+  it('an anchor without neighbours answers [] before the filter is asked', async () => {
+    const index = (brain as any).metadataIndex
+    const within = vi.spyOn(index, 'filterIdsWithin')
+    try {
+      const results = await brain.find({
+        connected: { from: 'noise-0', direction: 'out' },
+        where: { kind: 'note' },
+        limit: 10
+      })
+      expect(results).toEqual([])
+      expect(within).not.toHaveBeenCalled()
+    } finally {
+      within.mockRestore()
+    }
+  })
+})