diff --git a/src/brainy.ts b/src/brainy.ts index 6be20683..59eeabcc 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -146,6 +146,7 @@ import { resolveJsHnswConfig, DEFAULT_RECALL } from './utils/recallPreset.js' import * as fs from 'node:fs' import * as os from 'node:os' import { Db, type DbHost, type HistoricalQueryHandle } from './db/db.js' +import { entityMatchesFind } from './db/whereMatcher.js' import { importGraph, isPortableGraph, @@ -4706,7 +4707,7 @@ export class Brainy implements BrainyInterface { const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id) const startTime = Date.now() - const result = await (async () => { + let result = await (async () => { let results: Result[] = [] // Distinguish between search criteria (need vector search) and filter criteria (metadata only) @@ -5140,6 +5141,24 @@ export class Brainy implements BrainyInterface { return results.slice(finalOffset, finalOffset + limit) })() + // Index-integrity guard — applied ONCE here so every find() path (metadata, + // vector, text, proximity, graph) is covered uniformly. The indexes are + // acceleration structures; the loaded entity is ground truth. Re-validate + // each result against the query predicate so a stale or cross-bucket index + // entry — e.g. an id left in a field-value posting by a delete or an + // `update({ field: undefined })` — can never surface an entity that does not + // actually match `type`/`subtype`/`where`/`service`/`excludeVFS`. This is + // the live-path mirror of the historical path's per-candidate + // `entityMatchesFind` (db.ts). On a healthy index it is a no-op; on a + // corrupted one it drops the bad row instead of returning a phantom. + if (result.length > 0) { + result = result.filter( + (r) => + r.entity != null && + entityMatchesFind(r.entity as unknown as Entity, params as unknown as FindParams) + ) + } + // includeVectors — opt-in vector hydration. Default (false) keeps the perf // contract: every result path above builds entities via the metadata-only // fast path, so `entity.vector` is the empty stub. When requested, fetch the diff --git a/tests/unit/brainy/find-index-integrity-guard.test.ts b/tests/unit/brainy/find-index-integrity-guard.test.ts new file mode 100644 index 00000000..30cfdf1b --- /dev/null +++ b/tests/unit/brainy/find-index-integrity-guard.test.ts @@ -0,0 +1,101 @@ +/** + * @module find-index-integrity-guard.test + * @description Regression for the "phantom row" class (Venue 2026-06-24): a + * corrupt/stale metadata-index posting must never surface an entity that does + * not actually match the find() predicate. + * + * The metadata index is an acceleration structure; the loaded entity is ground + * truth. find() re-validates every result against `type`/`subtype`/`where`/ + * `service` at a single egress chokepoint (the live-path mirror of the + * historical path's `entityMatchesFind`). These tests inject a poisoned + * `getIdsForFilter` (the exact failure mode seen in production: a native index + * returns an id whose record matches NEITHER the type nor the where filter) and + * assert the phantom is dropped while the genuine matches survive. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +describe('find() index-integrity guard (phantom row class)', () => { + let brain: Brainy + let staffId: string + let timeslotId: string + let customerId: string + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + + // A real staff Person (the only legitimate match for the roster query). + staffId = await brain.add({ + data: 'Kristine Perry', + type: NounType.Person, + metadata: { entityType: 'staff', firstName: 'Kristine', lastName: 'Perry' } + }) + // A timeslot Event — matches NEITHER type:Person NOR where.entityType:'staff' + // (the production phantom was exactly this: a NounType.Event timeslot). + timeslotId = await brain.add({ + data: 'kitten-experience 2026-08-19 12:00', + type: NounType.Event, + metadata: { entityType: 'timeslot', date: '2026-08-19', startTime: '12:00' } + }) + // A Person that matches type:Person but NOT where.entityType:'staff' — proves + // the WHERE leg of the guard, not just the structural type leg. + customerId = await brain.add({ + data: 'Walk-in Customer', + type: NounType.Person, + metadata: { entityType: 'customer', firstName: 'Walk-in' } + }) + }) + + it('healthy index: the discriminant query returns only the staff Person', async () => { + const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) + expect(rows.map((r) => r.id)).toEqual([staffId]) + }) + + it('drops a phantom of the WRONG TYPE that a corrupt index leaks into the where bucket', async () => { + // Poison the metadata index so it returns the timeslot's id for the staff + // query — the precise production corruption (a cross-bucket/stale posting). + const mi = (brain as any).metadataIndex + const original = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any): Promise => { + const ids = await original(filter) + return ids.includes(timeslotId) ? ids : [...ids, timeslotId] + } + + const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) + + // The timeslot (NounType.Event, entityType:'timeslot') must NOT appear. + expect(rows.map((r) => r.id)).not.toContain(timeslotId) + // The genuine staff row survives. + expect(rows.map((r) => r.id)).toEqual([staffId]) + + mi.getIdsForFilter = original + }) + + it('drops a phantom of the RIGHT TYPE but WRONG where-value (exercises the where leg)', async () => { + // The customer is a Person (passes type) but entityType:'customer' (fails + // where). A corrupt index leaking it into the 'staff' bucket must still be + // rejected by the where re-validation. + const mi = (brain as any).metadataIndex + const original = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any): Promise => { + const ids = await original(filter) + return ids.includes(customerId) ? ids : [...ids, customerId] + } + + const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) + + expect(rows.map((r) => r.id)).not.toContain(customerId) + expect(rows.map((r) => r.id)).toEqual([staffId]) + + mi.getIdsForFilter = original + }) + + it('a healthy query for the phantom\'s own bucket still returns it (guard is not over-eager)', async () => { + // Sanity: the guard must only drop genuine non-matches. Querying the + // timeslot's real bucket returns it normally. + const rows = await brain.find({ type: NounType.Event, where: { entityType: 'timeslot' }, limit: 100 }) + expect(rows.map((r) => r.id)).toEqual([timeslotId]) + }) +})