feat(find): field projection — fields resolve from the column store, not the record

A list view that shows a title and a slug hydrates the whole record for every
row, document bodies included, and discards almost all of it. find/get({ fields })
names what is wanted; the column store serves it; the canonical record is opened
only for fields the index cannot supply.

The provider grows an optional getScalarsForIds(ids, fields) door, batched: it
walks each column ONCE and picks out every requested id, rather than re-walking
per row. The column store grows the primitive that was missing — valuesForIds —
because every other read door there answers which entities have a value, and a
projection asks the opposite.

It reads the COLUMN store, never the sparse index: the column keeps raw values,
the sparse index keeps a bucketed form built for range queries, and a projection
served from the latter would return a value that differs from the record's. A
field the column cannot serve is omitted rather than approximated — omission
costs a read, a wrong value is a wrong answer nobody can see.

Two laws the pins hold: fields absent is byte-identical to today, and a missing
field is simply absent rather than an error — so this path deliberately avoids
the strict address resolver, whose UnresolvableFieldError is right for orderBy
and wrong here.

related() takes no fields: a Relation carries from/to as ids and hydrates no
record, so the param would be decorative.

tests/integration/find-fields-projection.test.ts

@@ -0,0 +1,224 @@
+/**
+ * @module tests/integration/find-fields-projection
+ * @description **Field projection** — `find/get({ fields })` returns only the
+ * named fields, and serves them from the index when it can.
+ *
+ * A list view that shows a title and a slug does not need the document body,
+ * yet without a projection every row hydrates its whole record and discards
+ * almost all of it. These pins hold the two halves of the fix:
+ *
+ * **The answer.** A projected row is a SUBSET of the full row — for every
+ * requested field, the projected value equals the value the same query returns
+ * unprojected. Absent `fields` is byte-identical to today. A requested field the
+ * entity does not carry is simply absent, never an error. `system.*` resolves to
+ * the engine scalar, a bare name to the user's metadata.
+ *
+ * **The cost.** When every requested field is index-served, the canonical
+ * record is never opened — asserted by counting reads, not by timing them, so
+ * it cannot flake into a false green. When one requested field is NOT
+ * index-served (a body field, or a bucketed timestamp), exactly the owing rows
+ * are read and the rest are still served from the index.
+ */
+import { describe, it, expect, beforeAll, vi } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType } from '../../src/types/graphTypes'
+import { generateTestVector } from '../helpers/test-factory'
+
+/** Rows carrying a title, a slug, and a large body nobody wants in a list. */
+const ROWS = 12
+const BODY = 'x'.repeat(4096)
+
+describe('find/get({ fields }) — projection', () => {
+  let brain: Brainy<any>
+  const ids: string[] = []
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    for (let i = 0; i < ROWS; i++) {
+      ids.push(
+        await brain.add({
+          id: `post-${i}`,
+          data: `post ${i}`,
+          type: NounType.Thing,
+          metadata: {
+            kind: 'post',
+            title: `Title ${i}`,
+            slug: `slug-${i}`,
+            rank: i,
+            body: BODY,
+            // Only some rows carry this, so "missing is absent" is exercised
+            // by real data rather than by a name nothing ever had.
+            ...(i % 2 === 0 ? { featured: true } : {})
+          },
+          vector: generateTestVector()
+        })
+      )
+    }
+    // Persist so the column store holds the values a projection reads from.
+    await brain.flush()
+  })
+
+  /** Count canonical record reads for one call. */
+  const countingReads = async <R>(body: () => Promise<R>): Promise<{ out: R; reads: number }> => {
+    const spy = vi.spyOn(brain as any, 'batchGet')
+    try {
+      const out = await body()
+      const reads = spy.mock.calls.reduce(
+        (n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0),
+        0
+      )
+      return { out, reads }
+    } finally {
+      spy.mockRestore()
+    }
+  }
+
+  it('absent fields is byte-identical to today', async () => {
+    const params = { where: { kind: 'post' }, limit: 5 }
+    const a = await brain.find({ ...params })
+    const b = await brain.find({ ...params, fields: undefined })
+    expect(JSON.stringify(b)).toBe(JSON.stringify(a))
+  })
+
+  it('a projected row is a SUBSET of the full row, field for field', async () => {
+    const shapes: Array<Record<string, unknown>> = [
+      { where: { kind: 'post' }, limit: 6 },
+      { where: { kind: 'post' }, limit: 6, offset: 3 },
+      { where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 },
+      { where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 }
+    ]
+    for (const shape of shapes) {
+      const full = await brain.find(shape as never)
+      const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never)
+      expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id))
+      for (let i = 0; i < full.length; i++) {
+        const fullMeta = (full[i].entity.metadata ?? {}) as Record<string, unknown>
+        const projMeta = (projected[i].entity.metadata ?? {}) as Record<string, unknown>
+        expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title)
+        expect(projMeta.slug).toEqual(fullMeta.slug)
+      }
+    }
+  })
+
+  it('returns ONLY the named fields — the body never rides along', async () => {
+    const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 })
+    expect(rows).toHaveLength(4)
+    for (const r of rows) {
+      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
+      expect(Object.keys(meta)).toEqual(['title'])
+      expect(meta.body).toBeUndefined()
+      // Identity always survives a projection: a row you cannot identify is
+      // not a row.
+      expect(typeof r.id).toBe('string')
+      expect(r.entity.id).toBe(r.id)
+    }
+  })
+
+  it('a missing field is simply ABSENT — never an error', async () => {
+    // `featured` exists on half the rows; `no-such-field` on none. Neither
+    // throws, and neither appears as an explicit undefined.
+    const rows = await brain.find({
+      where: { kind: 'post' },
+      fields: ['title', 'featured', 'no-such-field'],
+      limit: ROWS
+    })
+    expect(rows.length).toBeGreaterThan(0)
+    let withFeatured = 0
+    for (const r of rows) {
+      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
+      expect('no-such-field' in meta).toBe(false)
+      if ('featured' in meta) withFeatured += 1
+    }
+    // Real data, not a name nothing ever had: some rows carry it, some do not.
+    expect(withFeatured).toBeGreaterThan(0)
+    expect(withFeatured).toBeLessThan(rows.length)
+  })
+
+  it('a strict address resolver is NOT on this path', async () => {
+    // orderBy throws UnresolvableFieldError for an unknown user key, because a
+    // typo there silently changes the order. A projection must not inherit that
+    // strictness: the honest answer to "give me this if you have it" is silence.
+    await expect(
+      brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 })
+    ).resolves.toBeInstanceOf(Array)
+  })
+
+  it('system.* resolves to the engine scalar, a bare name to user metadata', async () => {
+    const full = await brain.find({ where: { kind: 'post' }, limit: 3 })
+    const rows = await brain.find({
+      where: { kind: 'post' },
+      fields: ['system.createdAt', 'title'],
+      limit: 3
+    })
+    for (let i = 0; i < rows.length; i++) {
+      expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt)
+      const meta = (rows[i].entity.metadata ?? {}) as Record<string, unknown>
+      expect(meta.title).toEqual((full[i].entity.metadata as any).title)
+      // The engine scalar lands at the top level, not in the metadata bag —
+      // the two address spaces never shadow each other.
+      expect('system.createdAt' in meta).toBe(false)
+      expect('createdAt' in meta).toBe(false)
+    }
+  })
+
+  it('reads NO canonical record when every requested field is index-served', async () => {
+    // The cost pin, counted rather than timed. `title` and `slug` are ordinary
+    // indexed user fields, so the index can serve them exactly.
+    const { out, reads } = await countingReads(() =>
+      brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS })
+    )
+    expect(out.length).toBeGreaterThan(0)
+    expect(reads).toBe(0)
+  })
+
+  it('reads records only for the rows that owe an un-served field', async () => {
+    // `body` is not a scalar the index serves, so the record must be opened —
+    // but the projection still returns only the named fields.
+    const { out, reads } = await countingReads(() =>
+      brain.find({ where: { kind: 'post' }, fields: ['title', 'body'], limit: 4 })
+    )
+    expect(out).toHaveLength(4)
+    expect(reads).toBe(4)
+    for (const r of out) {
+      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
+      expect(meta.body).toBe(BODY)
+      expect(Object.keys(meta).sort()).toEqual(['body', 'title'])
+    }
+  })
+
+  it('get({ fields }) projects a single row through the same seam', async () => {
+    const full = await brain.get(ids[0])
+    const projected = await brain.get(ids[0], { fields: ['title', 'slug'] })
+    expect(projected).not.toBeNull()
+    expect(projected!.id).toBe(full!.id)
+    const fullMeta = (full!.metadata ?? {}) as Record<string, unknown>
+    const projMeta = (projected!.metadata ?? {}) as Record<string, unknown>
+    expect(projMeta.title).toEqual(fullMeta.title)
+    expect(projMeta.slug).toEqual(fullMeta.slug)
+    expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title'])
+    expect((projected as any).body).toBeUndefined()
+  })
+
+  it('get({ fields }) reads no record when the index serves the fields', async () => {
+    const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] }))
+    expect(reads).toBe(0)
+  })
+
+  it('the provider door serves only what it can serve EXACTLY', async () => {
+    // The bucketed timestamps are indexed at minute precision for range
+    // queries. The door must omit them rather than hand back a bucket that
+    // differs from the record — omission costs a read, a wrong value is a wrong
+    // answer nobody can see.
+    const index = (brain as any).metadataIndex
+    const served = await index.getScalarsForIds(ids.slice(0, 3), [
+      'title',
+      'system.createdAt'
+    ])
+    expect(served.size).toBeGreaterThan(0)
+    for (const [, row] of served) {
+      expect('title' in row).toBe(true)
+      expect('system.createdAt' in row).toBe(false)
+    }
+  })
+})