open-brainy/tests/integration/lens-consistency.test.ts

/**
 * @module tests/integration/lens-consistency
 * @description The three metadata "lenses" over one corpus must agree with
 * canonical ground truth id-for-id, warm AND after a cold reopen:
 *   - combined:      find({ type: T, where: { 'system.subtype': S } })
 *   - subtype-only:  find({ where: { 'system.subtype': S } })
 *   - type-only:     find({ type: T })
 * Ported from the fresh-brain probe that closed the type+subtype lens-drop
 * investigation (a restored pre-8.2.2 torn capture had entities visible to the
 * subtype-only lens but dropped by the combined lens — "0 of 2 migrated, all
 * gates green"). The corpus is seeded through the REAL write API — never
 * restored bytes — which is what made the original datapoint decisive. The
 * invariants: every lens matches an unfiltered canonical scan exactly (no
 * missing ids, no extras) and combined ⊆ subtype-only always holds.
 */
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'

/** The corpus: 7 (type, subtype) pairs, uneven counts, incl. the incident's 2-of-a-pair shape. */
const CORPUS: Array<{ type: string; subtype: string; count: number }> = [
  { type: 'proposition', subtype: 'decision', count: 2 }, // the incident shape: "0 of 2"
  { type: 'concept', subtype: 'decision', count: 3 },
  { type: 'task', subtype: 'decision', count: 2 },
  { type: 'concept', subtype: 'action', count: 4 },
  { type: 'message', subtype: 'note', count: 5 },
  { type: 'message', subtype: 'ship', count: 3 },
  { type: 'document', subtype: 'guide', count: 4 }
]

/** Canonical ground truth: unfiltered enumeration, post-filtered IN THE TEST. */
async function groundTruth(
  brain: any,
  match: { type?: string; subtype?: string }
): Promise<Set<string>> {
  const ids = new Set<string>()
  let cursor: string | undefined
  for (;;) {
    const page = await brain.storage.getNounsWithPagination({ limit: 500, cursor })
    for (const noun of page.items) {
      // Hydrated shape: `type`/`subtype` are TOP-LEVEL; `metadata` holds only
      // custom user fields (vfsType is one — the VFS plumbing marker).
      const n = noun as any
      if (n.metadata?.vfsType) continue // VFS plumbing is not corpus
      if (!n.type || !n.subtype) continue
      if (match.type && n.type !== match.type) continue
      if (match.subtype && n.subtype !== match.subtype) continue
      ids.add(n.id)
    }
    if (!page.hasMore) break
    cursor = page.nextCursor
  }
  return ids
}

const idSet = (results: Array<{ id: string }>): Set<string> => new Set(results.map((r) => r.id))

/** Every lens vs ground truth, id-for-id, for every pair in the corpus. */
async function assertAllLenses(brain: any): Promise<void> {
  const types = [...new Set(CORPUS.map((c) => c.type))]
  const subtypes = [...new Set(CORPUS.map((c) => c.subtype))]

  for (const { type, subtype } of CORPUS) {
    // system.subtype — subtype is an add()/update() param (an engine scalar),
    // never a user metadata field; bare 'subtype' now addresses the user's
    // own metadata bag under the sealed field-addressing law.
    const combined = idSet(await brain.find({ type, where: { 'system.subtype': subtype }, limit: 1000 }))
    const subtypeOnly = idSet(await brain.find({ where: { 'system.subtype': subtype }, limit: 1000 }))
    const truthPair = await groundTruth(brain, { type, subtype })
    const truthSubtype = await groundTruth(brain, { subtype })

    expect([...combined].sort()).toEqual([...truthPair].sort()) // no drops, no extras
    expect([...subtypeOnly].sort()).toEqual([...truthSubtype].sort())
    for (const id of combined) expect(subtypeOnly.has(id)).toBe(true) // combined ⊆ subtype-only
  }

  for (const type of types) {
    const typeOnly = idSet(await brain.find({ type, limit: 1000 }))
    const truthType = await groundTruth(brain, { type })
    expect([...typeOnly].sort()).toEqual([...truthType].sort())
  }

  // Count cross-check against the corpus definition itself.
  for (const subtype of subtypes) {
    const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0)
    const got = (await brain.find({ where: { 'system.subtype': subtype }, limit: 1000 })).length
    expect(got).toBe(expected)
  }
}

describe('lens consistency — combined vs subtype-only vs canonical ground truth', () => {
  let dir: string
  let brain: any

  beforeAll(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-lens-'))
    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
    await brain.init()
    // Seed through the REAL write API — never restored bytes.
    let i = 0
    for (const { type, subtype, count } of CORPUS) {
      for (let k = 0; k < count; k++) {
        await brain.add({ data: `${type} ${subtype} ${i++}`, type, subtype, metadata: { k } })
      }
    }
    await brain.flush()
  })
  afterAll(async () => {
    await brain.close?.().catch(() => {})
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('WARM: all lenses agree with ground truth id-for-id', async () => {
    await assertAllLenses(brain)
  })

  it('COLD REOPEN: all lenses still agree after close + reopen from disk', async () => {
    await brain.close()
    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
    await brain.init()
    await assertAllLenses(brain)
  })

  it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => {
    // The historical cross-bucket-staleness path: change (concept, action) -> (task, review).
    const victims = await brain.find({ type: 'concept', where: { 'system.subtype': 'action' }, limit: 1 })
    expect(victims.length).toBe(1)
    const id = victims[0].id
    await brain.update({ id, type: 'task', subtype: 'review' })

    const oldCombined = idSet(await brain.find({ type: 'concept', where: { 'system.subtype': 'action' }, limit: 1000 }))
    expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets
    const newCombined = idSet(await brain.find({ type: 'task', where: { 'system.subtype': 'review' }, limit: 1000 }))
    expect(newCombined.has(id)).toBe(true) // posted to the new buckets
    const subtypeOnly = idSet(await brain.find({ where: { 'system.subtype': 'review' }, limit: 1000 }))
    expect(subtypeOnly.has(id)).toBe(true)
  })
})