David Snelling
69bda5b7cb
The seam hydrates the metadata and graph page paths; a vector or text leg builds its own entities and is trimmed after the integrity guard instead. That is a COST difference, and this pin exists so it can never quietly become an ANSWER difference.
261 lines
11 KiB
TypeScript
261 lines
11 KiB
TypeScript
/**
|
|
* @module tests/integration/find-fields-projection
|
|
* @description **Field projection** — `find/get({ fields })` returns only the
|
|
* named fields, and serves them from the index when it can.
|
|
*
|
|
* A list view that shows a title and a slug does not need the document body,
|
|
* yet without a projection every row hydrates its whole record and discards
|
|
* almost all of it. These pins hold the two halves of the fix:
|
|
*
|
|
* **The answer.** A projected row is a SUBSET of the full row — for every
|
|
* requested field, the projected value equals the value the same query returns
|
|
* unprojected. Absent `fields` is byte-identical to today. A requested field the
|
|
* entity does not carry is simply absent, never an error. `system.*` resolves to
|
|
* the engine scalar, a bare name to the user's metadata.
|
|
*
|
|
* **The cost.** When every requested field is index-served, the canonical
|
|
* record is never opened — asserted by counting reads, not by timing them, so
|
|
* it cannot flake into a false green. When one requested field is NOT
|
|
* index-served (a body field, or a bucketed timestamp), exactly the owing rows
|
|
* are read and the rest are still served from the index.
|
|
*/
|
|
import { describe, it, expect, beforeAll, vi } from 'vitest'
|
|
import { Brainy } from '../../src/brainy'
|
|
import { NounType } from '../../src/types/graphTypes'
|
|
import { generateTestVector } from '../helpers/test-factory'
|
|
|
|
/** Rows carrying a title, a slug, and a large body nobody wants in a list. */
|
|
const ROWS = 12
|
|
const BODY = 'x'.repeat(4096)
|
|
|
|
describe('find/get({ fields }) — projection', () => {
|
|
let brain: Brainy<any>
|
|
const ids: string[] = []
|
|
|
|
beforeAll(async () => {
|
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
await brain.init()
|
|
for (let i = 0; i < ROWS; i++) {
|
|
ids.push(
|
|
await brain.add({
|
|
id: `post-${i}`,
|
|
data: `post ${i}`,
|
|
type: NounType.Thing,
|
|
metadata: {
|
|
kind: 'post',
|
|
title: `Title ${i}`,
|
|
slug: `slug-${i}`,
|
|
rank: i,
|
|
body: BODY,
|
|
// Only some rows carry this, so "missing is absent" is exercised
|
|
// by real data rather than by a name nothing ever had.
|
|
...(i % 2 === 0 ? { featured: true } : {})
|
|
},
|
|
vector: generateTestVector()
|
|
})
|
|
)
|
|
}
|
|
// Persist so the column store holds the values a projection reads from.
|
|
await brain.flush()
|
|
})
|
|
|
|
/** Count canonical record reads for one call. */
|
|
const countingReads = async <R>(body: () => Promise<R>): Promise<{ out: R; reads: number }> => {
|
|
const spy = vi.spyOn(brain as any, 'batchGet')
|
|
try {
|
|
const out = await body()
|
|
const reads = spy.mock.calls.reduce(
|
|
(n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0),
|
|
0
|
|
)
|
|
return { out, reads }
|
|
} finally {
|
|
spy.mockRestore()
|
|
}
|
|
}
|
|
|
|
it('absent fields is byte-identical to today', async () => {
|
|
const params = { where: { kind: 'post' }, limit: 5 }
|
|
const a = await brain.find({ ...params })
|
|
const b = await brain.find({ ...params, fields: undefined })
|
|
expect(JSON.stringify(b)).toBe(JSON.stringify(a))
|
|
})
|
|
|
|
it('a projected row is a SUBSET of the full row, field for field', async () => {
|
|
const shapes: Array<Record<string, unknown>> = [
|
|
{ where: { kind: 'post' }, limit: 6 },
|
|
{ where: { kind: 'post' }, limit: 6, offset: 3 },
|
|
{ where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 },
|
|
{ where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 }
|
|
]
|
|
for (const shape of shapes) {
|
|
const full = await brain.find(shape as never)
|
|
const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never)
|
|
expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id))
|
|
for (let i = 0; i < full.length; i++) {
|
|
const fullMeta = (full[i].entity.metadata ?? {}) as Record<string, unknown>
|
|
const projMeta = (projected[i].entity.metadata ?? {}) as Record<string, unknown>
|
|
expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title)
|
|
expect(projMeta.slug).toEqual(fullMeta.slug)
|
|
}
|
|
}
|
|
})
|
|
|
|
it('returns ONLY the named fields — the body never rides along', async () => {
|
|
const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 })
|
|
expect(rows).toHaveLength(4)
|
|
for (const r of rows) {
|
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
|
expect(Object.keys(meta)).toEqual(['title'])
|
|
expect(meta.body).toBeUndefined()
|
|
// Identity always survives a projection: a row you cannot identify is
|
|
// not a row.
|
|
expect(typeof r.id).toBe('string')
|
|
expect(r.entity.id).toBe(r.id)
|
|
}
|
|
})
|
|
|
|
it('a missing field is simply ABSENT — never an error', async () => {
|
|
// `featured` exists on half the rows; `no-such-field` on none. Neither
|
|
// throws, and neither appears as an explicit undefined.
|
|
const rows = await brain.find({
|
|
where: { kind: 'post' },
|
|
fields: ['title', 'featured', 'no-such-field'],
|
|
limit: ROWS
|
|
})
|
|
expect(rows.length).toBeGreaterThan(0)
|
|
let withFeatured = 0
|
|
for (const r of rows) {
|
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
|
expect('no-such-field' in meta).toBe(false)
|
|
if ('featured' in meta) withFeatured += 1
|
|
}
|
|
// Real data, not a name nothing ever had: some rows carry it, some do not.
|
|
expect(withFeatured).toBeGreaterThan(0)
|
|
expect(withFeatured).toBeLessThan(rows.length)
|
|
})
|
|
|
|
it('a strict address resolver is NOT on this path', async () => {
|
|
// orderBy throws UnresolvableFieldError for an unknown user key, because a
|
|
// typo there silently changes the order. A projection must not inherit that
|
|
// strictness: the honest answer to "give me this if you have it" is silence.
|
|
await expect(
|
|
brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 })
|
|
).resolves.toBeInstanceOf(Array)
|
|
})
|
|
|
|
it('system.* resolves to the engine scalar, a bare name to user metadata', async () => {
|
|
const full = await brain.find({ where: { kind: 'post' }, limit: 3 })
|
|
const rows = await brain.find({
|
|
where: { kind: 'post' },
|
|
fields: ['system.createdAt', 'title'],
|
|
limit: 3
|
|
})
|
|
for (let i = 0; i < rows.length; i++) {
|
|
expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt)
|
|
const meta = (rows[i].entity.metadata ?? {}) as Record<string, unknown>
|
|
expect(meta.title).toEqual((full[i].entity.metadata as any).title)
|
|
// The engine scalar lands at the top level, not in the metadata bag —
|
|
// the two address spaces never shadow each other.
|
|
expect('system.createdAt' in meta).toBe(false)
|
|
expect('createdAt' in meta).toBe(false)
|
|
}
|
|
})
|
|
|
|
it('reads NO canonical record when every requested field is index-served', async () => {
|
|
// The cost pin, counted rather than timed. `title` and `slug` are ordinary
|
|
// indexed user fields, so the index can serve them exactly.
|
|
const { out, reads } = await countingReads(() =>
|
|
brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS })
|
|
)
|
|
expect(out.length).toBeGreaterThan(0)
|
|
expect(reads).toBe(0)
|
|
})
|
|
|
|
it('reads records only for the fields the column cannot serve', async () => {
|
|
// `system.data` is NOT a column the store holds (verified against
|
|
// getIndexedFields), so the record must be opened for it — while `title`,
|
|
// which the column does hold, still comes from the index.
|
|
const { out, reads } = await countingReads(() =>
|
|
brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 })
|
|
)
|
|
expect(out).toHaveLength(4)
|
|
expect(reads).toBe(4)
|
|
for (const r of out) {
|
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
|
expect(Object.keys(meta)).toEqual(['title'])
|
|
expect(typeof (r.entity as any).data).toBe('string')
|
|
}
|
|
})
|
|
|
|
it('a large field the column DOES hold costs no record read', async () => {
|
|
// Worth pinning because it is the venue case: the body is column-served on
|
|
// this engine, so a list that projects around it pays nothing for it, and
|
|
// a list that projects it still pays no record read.
|
|
const { reads } = await countingReads(() =>
|
|
brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 })
|
|
)
|
|
expect(reads).toBe(0)
|
|
})
|
|
|
|
it('projects a vector-leg find too — the ANSWER is uniform, only the cost is not', async () => {
|
|
// The seam hydrates the metadata and graph page paths. A vector or text leg
|
|
// builds its own entities, so those rows are trimmed after the integrity
|
|
// guard instead. That difference is a COST difference, and this pin exists
|
|
// so it can never quietly become an ANSWER difference.
|
|
const rows = await brain.find({ query: 'post', fields: ['title'], limit: 3 })
|
|
for (const r of rows) {
|
|
const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
|
|
expect(Object.keys(meta)).toEqual(['title'])
|
|
expect(meta.body).toBeUndefined()
|
|
expect(r.entity.id).toBe(r.id)
|
|
}
|
|
})
|
|
|
|
it('get({ fields }) projects a single row through the same seam', async () => {
|
|
const full = await brain.get(ids[0])
|
|
const projected = await brain.get(ids[0], { fields: ['title', 'slug'] })
|
|
expect(projected).not.toBeNull()
|
|
expect(projected!.id).toBe(full!.id)
|
|
const fullMeta = (full!.metadata ?? {}) as Record<string, unknown>
|
|
const projMeta = (projected!.metadata ?? {}) as Record<string, unknown>
|
|
expect(projMeta.title).toEqual(fullMeta.title)
|
|
expect(projMeta.slug).toEqual(fullMeta.slug)
|
|
expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title'])
|
|
expect((projected as any).body).toBeUndefined()
|
|
})
|
|
|
|
it('get({ fields }) reads no record when the index serves the fields', async () => {
|
|
const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] }))
|
|
expect(reads).toBe(0)
|
|
})
|
|
|
|
it('the door serves EXACT values — the column, never the bucketed index', async () => {
|
|
// The sparse index buckets `system.createdAt` to the minute for range
|
|
// queries; the column store keeps raw ms. Serving a projection from the
|
|
// former would hand back a value that differs from the record's, so the
|
|
// door reads the column — and this pin is what proves which one it read.
|
|
const index = (brain as any).metadataIndex
|
|
const sample = ids.slice(0, 3)
|
|
const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt'])
|
|
expect(served.size).toBe(sample.length)
|
|
for (const id of sample) {
|
|
const row = served.get(id)!
|
|
const record = await brain.get(id)
|
|
expect(row.title).toEqual((record!.metadata as any).title)
|
|
// Exact to the millisecond — a bucketed value would be rounded down to
|
|
// the minute and this would fail.
|
|
expect(row['system.createdAt']).toEqual((record as any).createdAt)
|
|
}
|
|
})
|
|
|
|
it('a field the column store does not hold is OMITTED, not approximated', async () => {
|
|
const index = (brain as any).metadataIndex
|
|
const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data'])
|
|
for (const [, row] of served) {
|
|
expect('title' in row).toBe(true)
|
|
// Omission is what makes the caller read the record for it.
|
|
expect('system.data' in row).toBe(false)
|
|
}
|
|
})
|
|
})
|