open-brainy/tests/integration/find-fields-projection.test.ts

/**
 * @module tests/integration/find-fields-projection
 * @description **Field projection** — `find/get({ fields })` returns only the
 * named fields, and serves them from the index when it can.
 *
 * A list view that shows a title and a slug does not need the document body,
 * yet without a projection every row hydrates its whole record and discards
 * almost all of it. These pins hold the two halves of the fix:
 *
 * **The answer.** A projected row is a SUBSET of the full row — for every
 * requested field, the projected value equals the value the same query returns
 * unprojected. Absent `fields` is byte-identical to today. A requested field the
 * entity does not carry is simply absent, never an error. `system.*` resolves to
 * the engine scalar, a bare name to the user's metadata.
 *
 * **The cost.** When every requested field is index-served, the canonical
 * record is never opened — asserted by counting reads, not by timing them, so
 * it cannot flake into a false green. When one requested field is NOT
 * index-served (a body field, or a bucketed timestamp), exactly the owing rows
 * are read and the rest are still served from the index.
 */
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
import { Brainy } from '../../src/brainy'
import { NounType } from '../../src/types/graphTypes'
import { generateTestVector } from '../helpers/test-factory'

/** Rows carrying a title, a slug, and a large body nobody wants in a list. */
const ROWS = 12
const BODY = 'x'.repeat(4096)

describe('find/get({ fields }) — projection', () => {
  let brain: Brainy<any>
  const ids: string[] = []

  beforeAll(async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
    await brain.init()
    for (let i = 0; i < ROWS; i++) {
      ids.push(
        await brain.add({
          id: `post-${i}`,
          data: `post ${i}`,
          type: NounType.Thing,
          metadata: {
            kind: 'post',
            title: `Title ${i}`,
            slug: `slug-${i}`,
            rank: i,
            body: BODY,
            // Only some rows carry this, so "missing is absent" is exercised
            // by real data rather than by a name nothing ever had.
            ...(i % 2 === 0 ? { featured: true } : {})
          },
          vector: generateTestVector()
        })
      )
    }
    // Persist so the column store holds the values a projection reads from.
    await brain.flush()
  })

  afterAll(async () => {
    await brain.close()
  })

  /** Count canonical record reads for one call. */
  const countingReads = async <R>(body: () => Promise<R>): Promise<{ out: R; reads: number }> => {
    const spy = vi.spyOn(brain as any, 'batchGet')
    try {
      const out = await body()
      const reads = spy.mock.calls.reduce(
        (n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0),
        0
      )
      return { out, reads }
    } finally {
      spy.mockRestore()
    }
  }

  it('absent fields is byte-identical to today', async () => {
    const params = { where: { kind: 'post' }, limit: 5 }
    const a = await brain.find({ ...params })
    const b = await brain.find({ ...params, fields: undefined })
    expect(JSON.stringify(b)).toBe(JSON.stringify(a))
  })

  it('a projected row is a SUBSET of the full row, field for field', async () => {
    const shapes: Array<Record<string, unknown>> = [
      { where: { kind: 'post' }, limit: 6 },
      { where: { kind: 'post' }, limit: 6, offset: 3 },
      { where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 },
      { where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 }
    ]
    for (const shape of shapes) {
      const full = await brain.find(shape as never)
      const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never)
      expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id))
      for (let i = 0; i < full.length; i++) {
        const fullMeta = (full[i].entity.metadata ?? {}) as Record<string, unknown>
        const projMeta = (projected[i].entity.metadata ?? {}) as Record<string, unknown>
        expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title)
        expect(projMeta.slug).toEqual(fullMeta.slug)
      }
    }
  })

  it('returns ONLY the named fields — the body never rides along', async () => {
    const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 })
    expect(rows).toHaveLength(4)
    for (const r of rows) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect(Object.keys(meta)).toEqual(['title'])
      expect(meta.body).toBeUndefined()
      // Identity always survives a projection: a row you cannot identify is
      // not a row.
      expect(typeof r.id).toBe('string')
      expect(r.entity.id).toBe(r.id)
    }
  })

  it('a missing field is simply ABSENT — never an error', async () => {
    // `featured` exists on half the rows; `no-such-field` on none. Neither
    // throws, and neither appears as an explicit undefined.
    const rows = await brain.find({
      where: { kind: 'post' },
      fields: ['title', 'featured', 'no-such-field'],
      limit: ROWS
    })
    expect(rows.length).toBeGreaterThan(0)
    let withFeatured = 0
    for (const r of rows) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect('no-such-field' in meta).toBe(false)
      if ('featured' in meta) withFeatured += 1
    }
    // Real data, not a name nothing ever had: some rows carry it, some do not.
    expect(withFeatured).toBeGreaterThan(0)
    expect(withFeatured).toBeLessThan(rows.length)
  })

  it('a strict address resolver is NOT on this path', async () => {
    // orderBy throws UnresolvableFieldError for an unknown user key, because a
    // typo there silently changes the order. A projection must not inherit that
    // strictness: the honest answer to "give me this if you have it" is silence.
    await expect(
      brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 })
    ).resolves.toBeInstanceOf(Array)
  })

  it('system.* resolves to the engine scalar, a bare name to user metadata', async () => {
    const full = await brain.find({ where: { kind: 'post' }, limit: 3 })
    const rows = await brain.find({
      where: { kind: 'post' },
      fields: ['system.createdAt', 'title'],
      limit: 3
    })
    for (let i = 0; i < rows.length; i++) {
      expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt)
      const meta = (rows[i].entity.metadata ?? {}) as Record<string, unknown>
      expect(meta.title).toEqual((full[i].entity.metadata as any).title)
      // The engine scalar lands at the top level, not in the metadata bag —
      // the two address spaces never shadow each other.
      expect('system.createdAt' in meta).toBe(false)
      expect('createdAt' in meta).toBe(false)
    }
  })

  it('reads NO canonical record when every requested field is index-served', async () => {
    // The cost pin, counted rather than timed. `title` and `slug` are ordinary
    // indexed user fields, so the index can serve them exactly.
    const { out, reads } = await countingReads(() =>
      brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS })
    )
    expect(out.length).toBeGreaterThan(0)
    expect(reads).toBe(0)
  })

  it('reads records only for the fields the column cannot serve', async () => {
    // `system.data` is NOT a column the store holds (verified against
    // getIndexedFields), so the record must be opened for it — while `title`,
    // which the column does hold, still comes from the index.
    const { out, reads } = await countingReads(() =>
      brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 })
    )
    expect(out).toHaveLength(4)
    expect(reads).toBe(4)
    for (const r of out) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect(Object.keys(meta)).toEqual(['title'])
      expect(typeof (r.entity as any).data).toBe('string')
    }
  })

  it('a large field the column DOES hold costs no record read', async () => {
    // Worth pinning because it is the venue case: the body is column-served on
    // this engine, so a list that projects around it pays nothing for it, and
    // a list that projects it still pays no record read.
    const { reads } = await countingReads(() =>
      brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 })
    )
    expect(reads).toBe(0)
  })

  it('projects a vector-leg find too — the ANSWER is uniform, only the cost is not', async () => {
    // The seam hydrates the metadata and graph page paths. A vector or text leg
    // builds its own entities, so those rows are trimmed after the integrity
    // guard instead. That difference is a COST difference, and this pin exists
    // so it can never quietly become an ANSWER difference.
    const rows = await brain.find({ query: 'post', fields: ['title'], limit: 3 })
    for (const r of rows) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect(Object.keys(meta)).toEqual(['title'])
      expect(meta.body).toBeUndefined()
      expect(r.entity.id).toBe(r.id)
    }
  })

  it('get({ fields }) projects a single row through the same seam', async () => {
    const full = await brain.get(ids[0])
    const projected = await brain.get(ids[0], { fields: ['title', 'slug'] })
    expect(projected).not.toBeNull()
    expect(projected!.id).toBe(full!.id)
    const fullMeta = (full!.metadata ?? {}) as Record<string, unknown>
    const projMeta = (projected!.metadata ?? {}) as Record<string, unknown>
    expect(projMeta.title).toEqual(fullMeta.title)
    expect(projMeta.slug).toEqual(fullMeta.slug)
    expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title'])
    expect((projected as any).body).toBeUndefined()
  })

  it('get({ fields }) reads no record when the index serves the fields', async () => {
    const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] }))
    expect(reads).toBe(0)
  })

  it('the door serves EXACT values — the column, never the bucketed index', async () => {
    // The sparse index buckets `system.createdAt` to the minute for range
    // queries; the column store keeps raw ms. Serving a projection from the
    // former would hand back a value that differs from the record's, so the
    // door reads the column — and this pin is what proves which one it read.
    const index = (brain as any).metadataIndex
    const sample = ids.slice(0, 3)
    const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt'])
    expect(served.size).toBe(sample.length)
    for (const id of sample) {
      const row = served.get(id)!
      const record = await brain.get(id)
      expect(row.title).toEqual((record!.metadata as any).title)
      // Exact to the millisecond — a bucketed value would be rounded down to
      // the minute and this would fail.
      expect(row['system.createdAt']).toEqual((record as any).createdAt)
    }
  })

  it('a field the column store does not hold is OMITTED, not approximated', async () => {
    const index = (brain as any).metadataIndex
    const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data'])
    for (const [, row] of served) {
      expect('title' in row).toBe(true)
      // Omission is what makes the caller read the record for it.
      expect('system.data' in row).toBe(false)
    }
  })
})