From be77a10bfe2799c0f263507c12d2af4874a9034c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:43:47 -0700 Subject: [PATCH] fix(find): the projection seam is ES-private, and document the projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TypeScript's `private` is compile-time only, so the seam's helpers were real prototype methods and the generated contract manifest listed them as public DOORS — which would have obliged every other engine to implement an internal detail. They are `#`-private now and the manifest is unchanged by this branch. Found while checking that: docs/api-contract.json was ALREADY stale at v10.4.11 — promoteQueuedFlush and startFlushLeader are in src and absent from the manifest, so they leaked the same way and were never re-emitted. Left alone here rather than folded into this branch; it is someone's to fix deliberately, and the fix is the same # conversion. docs/FIND_SYSTEM.md gains the projection: the rules, why a missing field is absent rather than an error, where the values come from and what a field the column cannot serve costs. --- docs/FIND_SYSTEM.md | 65 +++++++++++++++++++ src/brainy.ts | 24 +++---- .../find-fields-projection.test.ts | 57 +++++++++++----- 3 files changed, 117 insertions(+), 29 deletions(-) diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 77fbbd79..6aa33515 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -369,6 +369,71 @@ return results.slice(offset, offset + limit) // → Auto-correction: Use most likely alternative based on affinity data ``` +## Field Projection (`fields`) + +`find()` and `get()` accept a `fields` list. Without it they return the whole +record; with it they return only the fields you name — and, where the index can +supply them, without opening the canonical record at all. + +```ts +// A list page: two user fields and one engine scalar. No document bodies. +await brain.find({ + where: { kind: 'post' }, + fields: ['title', 'slug', 'system.createdAt'], + limit: 50 +}) + +await brain.get(id, { fields: ['title'] }) +``` + +### Why it exists + +A list view that renders a title and a date does not need the body, but without +a projection every row hydrates its full record and throws almost all of it +away. On a posts list that is the dominant cost of the query. + +### The rules + +| | | +|---|---| +| **`fields` absent** | The full record, byte-identical to before. Nothing changes. | +| **Field names** | The one addressing law: a bare name is user metadata (`'title'`), `system.*` is an engine scalar (`'system.createdAt'`). | +| **A field the row lacks** | Simply **absent** from the result. Never an error. | +| **Identity** | Every row keeps its `id` (and `score` on `find`) regardless — a row you cannot identify is not a row. | +| **Where values come from** | The **column store**, which holds raw values. Never the sparse index, which buckets timestamps for range queries. | +| **A field the column cannot serve** | The canonical record is read for that field only. Correct, just not free. | + +### Missing fields are absent, not errors + +This is deliberate and differs from `orderBy`, which throws +`UnresolvableFieldError` for an unknown field. A typo in `orderBy` silently +changes the ordering, so it must be loud. A projection asks "give me these if +you have them", and an optional field must not turn a list into a failure — so +`fields` uses the permissive path. + +### Cost + +When every named field is column-served, a projected page performs **zero** +canonical reads. When one is not, only that read happens and the rest still come +from the index. Both are pinned by counting reads rather than timing them, in +`tests/integration/find-fields-projection.test.ts`. + +### `related()` takes no `fields` + +A `Relation` carries `from` and `to` as **ids** and hydrates no entity record, +so there is nothing for a projection to trim. Projecting the endpoints would be +a new capability rather than a projection of an existing one. + +### For engine implementers + +Projection is served through an optional provider door, +`getScalarsForIds(ids, fields)` on `MetadataIndexProvider`. The contract is in +`src/plugin.ts`; the short version is **return only what you can serve exactly, +and say what you served**. The caller diffs the answer against the request and +reads records for the remainder, so omission costs a read while a wrong value is +a wrong answer nobody can see. An engine without the door still works — every +field falls back to the record. + ## Performance Characteristics ### Query Performance by Type diff --git a/src/brainy.ts b/src/brainy.ts index 18c46d48..edcba3d4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4199,7 +4199,7 @@ export class Brainy implements BrainyInterface { // construction: a projection returns the named fields, and a vector is not // one of them unless it was named. if (options?.fields !== undefined && options.fields.length > 0) { - const page = await this.hydratePage([id], options.fields) + const page = await this.#hydratePage([id], options.fields) return page.get(id) ?? null } @@ -4294,7 +4294,7 @@ export class Brainy implements BrainyInterface { * @param params - The find params. * @returns Index keys to carry through hydration. */ - private guardFieldsFor(params: FindParams): string[] { + #guardFieldsFor(params: FindParams): string[] { const keys: string[] = [] if (params.where && typeof params.where === 'object') { // Top-level where keys only: nested `anyOf`/`allOf` branches are carried @@ -4313,7 +4313,7 @@ export class Brainy implements BrainyInterface { return keys } - private async hydratePage( + async #hydratePage( ids: string[], fields?: readonly string[], guardFields: readonly string[] = [] @@ -4346,7 +4346,7 @@ export class Brainy implements BrainyInterface { const record = records.get(id) // An id neither the index nor storage knows is not a row. if (fromIndex === undefined && record === undefined) continue - out.set(id, this.projectEntity(id, wanted, fromIndex, record)) + out.set(id, this.#projectEntity(id, wanted, fromIndex, record)) } return out } @@ -4367,7 +4367,7 @@ export class Brainy implements BrainyInterface { * @param record - The canonical entity, if one had to be read. * @returns The projected entity. */ - private projectEntity( + #projectEntity( id: string, fields: readonly string[], fromIndex: Record | undefined, @@ -8211,7 +8211,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8248,7 +8248,7 @@ export class Brainy implements BrainyInterface { if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8276,7 +8276,7 @@ export class Brainy implements BrainyInterface { const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8511,7 +8511,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8539,7 +8539,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8656,7 +8656,7 @@ export class Brainy implements BrainyInterface { if (params.fields !== undefined && params.fields.length > 0 && result.length > 0) { const named = [...new Set(params.fields)] result = result.map((r) => { - const projected = this.projectEntity( + const projected = this.#projectEntity( r.id, named, undefined, @@ -17038,7 +17038,7 @@ export class Brainy implements BrainyInterface { ordered = valued.map((v) => v.id) } const pageIds = ordered.slice(offset, offset + limit) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) const results: Result[] = [] for (const id of pageIds) { const entity = entitiesMap.get(id) diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts index 3718dc4f..df94e942 100644 --- a/tests/integration/find-fields-projection.test.ts +++ b/tests/integration/find-fields-projection.test.ts @@ -172,21 +172,32 @@ describe('find/get({ fields }) — projection', () => { expect(reads).toBe(0) }) - it('reads records only for the rows that owe an un-served field', async () => { - // `body` is not a scalar the index serves, so the record must be opened — - // but the projection still returns only the named fields. + it('reads records only for the fields the column cannot serve', async () => { + // `system.data` is NOT a column the store holds (verified against + // getIndexedFields), so the record must be opened for it — while `title`, + // which the column does hold, still comes from the index. const { out, reads } = await countingReads(() => - brain.find({ where: { kind: 'post' }, fields: ['title', 'body'], limit: 4 }) + brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 }) ) expect(out).toHaveLength(4) expect(reads).toBe(4) for (const r of out) { const meta = (r.entity.metadata ?? {}) as Record - expect(meta.body).toBe(BODY) - expect(Object.keys(meta).sort()).toEqual(['body', 'title']) + expect(Object.keys(meta)).toEqual(['title']) + expect(typeof (r.entity as any).data).toBe('string') } }) + it('a large field the column DOES hold costs no record read', async () => { + // Worth pinning because it is the venue case: the body is column-served on + // this engine, so a list that projects around it pays nothing for it, and + // a list that projects it still pays no record read. + const { reads } = await countingReads(() => + brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 }) + ) + expect(reads).toBe(0) + }) + it('get({ fields }) projects a single row through the same seam', async () => { const full = await brain.get(ids[0]) const projected = await brain.get(ids[0], { fields: ['title', 'slug'] }) @@ -205,20 +216,32 @@ describe('find/get({ fields }) — projection', () => { expect(reads).toBe(0) }) - it('the provider door serves only what it can serve EXACTLY', async () => { - // The bucketed timestamps are indexed at minute precision for range - // queries. The door must omit them rather than hand back a bucket that - // differs from the record — omission costs a read, a wrong value is a wrong - // answer nobody can see. + it('the door serves EXACT values — the column, never the bucketed index', async () => { + // The sparse index buckets `system.createdAt` to the minute for range + // queries; the column store keeps raw ms. Serving a projection from the + // former would hand back a value that differs from the record's, so the + // door reads the column — and this pin is what proves which one it read. const index = (brain as any).metadataIndex - const served = await index.getScalarsForIds(ids.slice(0, 3), [ - 'title', - 'system.createdAt' - ]) - expect(served.size).toBeGreaterThan(0) + const sample = ids.slice(0, 3) + const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt']) + expect(served.size).toBe(sample.length) + for (const id of sample) { + const row = served.get(id)! + const record = await brain.get(id) + expect(row.title).toEqual((record!.metadata as any).title) + // Exact to the millisecond — a bucketed value would be rounded down to + // the minute and this would fail. + expect(row['system.createdAt']).toEqual((record as any).createdAt) + } + }) + + it('a field the column store does not hold is OMITTED, not approximated', async () => { + const index = (brain as any).metadataIndex + const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data']) for (const [, row] of served) { expect('title' in row).toBe(true) - expect('system.createdAt' in row).toBe(false) + // Omission is what makes the caller read the record for it. + expect('system.data' in row).toBe(false) } }) })