diff --git a/src/brainy.ts b/src/brainy.ts index 3fe57053..18c46d48 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4193,6 +4193,16 @@ export class Brainy implements BrainyInterface { } // Route to metadata-only or full entity based on options + // A PROJECTED get goes through the same seam every list page uses, so a + // detail read of two scalars costs an index read rather than a record read. + // It is checked before `includeVectors` because the two are incompatible by + // construction: a projection returns the named fields, and a vector is not + // one of them unless it was named. + if (options?.fields !== undefined && options.fields.length > 0) { + const page = await this.hydratePage([id], options.fields) + return page.get(id) ?? null + } + const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) if (includeVectors) { @@ -4239,6 +4249,170 @@ export class Brainy implements BrainyInterface { * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean) * ``` */ + /** + * **The projection seam** — hydrate a page of ids under an optional `fields` + * projection, opening the canonical record only when the index cannot serve + * what was asked for. + * + * Without a projection this is exactly `batchGet`, byte for byte: the whole + * point is that `fields` absent changes nothing. + * + * With one, the order is: ask the index for the named scalars in a single + * batched door; see which requested fields it actually served; and read + * records ONLY if something is still missing — and only to fill those fields. + * A page whose every requested field is index-served performs zero canonical + * reads, which is the whole reason the door exists. + * + * `guardFields` are fetched ALONGSIDE the projection and trimmed off before + * the caller sees them. find()'s index-integrity guard re-validates every row + * against its own predicate, and it reads the entity to do so — so a row + * projected down to `title` would fail a `where: { kind }` it genuinely + * matches, and the whole page would vanish. The fields a filter names are + * fields the index can serve by definition, so carrying them costs nothing + * and keeps the guard honest. + * + * A field nothing can supply is simply absent from the row. That is the + * permissive law: a projection asks "these, if you have them", and an + * optional field must not turn a list into an exception. It deliberately does + * NOT route through the strict address resolver, which throws + * `UnresolvableFieldError` for an unknown key — that strictness is right for + * `orderBy`, where a typo silently changes the order, and wrong here, where + * the honest answer is "this row does not have that". + * + * @param ids - Canonical ids for the page. + * @param fields - The projection, or undefined for the full record. + * @returns `id → entity`, projected when `fields` was given. + */ + /** + * The index keys find()'s integrity guard reads when it re-validates a row. + * + * The guard calls `entityMatchesFind(entity, params)`, so a projected entity + * must still carry whatever the params constrain — otherwise a row that + * genuinely matches is dropped for lacking the evidence. These are fetched + * with the projection and trimmed off before the caller sees them. + * + * @param params - The find params. + * @returns Index keys to carry through hydration. + */ + private guardFieldsFor(params: FindParams): string[] { + const keys: string[] = [] + if (params.where && typeof params.where === 'object') { + // Top-level where keys only: nested `anyOf`/`allOf` branches are carried + // by their own keys when the guard walks them, and a filter whose + // evidence is missing keeps the row (the guard's own catch) rather than + // dropping it. + for (const key of Object.keys(params.where as Record)) { + if (key === 'anyOf' || key === 'allOf' || key === 'not') continue + keys.push(key) + } + } + if (params.type !== undefined) keys.push('system.type') + if (params.subtype !== undefined) keys.push('system.subtype') + if (params.service !== undefined) keys.push('system.service') + if (params.excludeVFS === true) keys.push('vfsType', 'isVFSEntity') + return keys + } + + private async hydratePage( + ids: string[], + fields?: readonly string[], + guardFields: readonly string[] = [] + ): Promise>> { + if (fields === undefined || fields.length === 0) return this.batchGet(ids) + + const wanted = [...new Set([...fields, ...guardFields])] + const provider = this.metadataIndex as unknown as MetadataIndexProvider + let served = new Map>() + if (typeof provider.getScalarsForIds === 'function') { + served = await provider.getScalarsForIds(ids, wanted) + } + + // Which ids still owe a field? Only those cost a record read, and a page + // that owes nothing costs none at all. + const owing: string[] = [] + for (const id of ids) { + const row = served.get(id) + if (row === undefined || wanted.some((f) => !(f in row))) owing.push(id) + } + + // The records are read for the OWED fields only; everything the index + // already served is used as-is, so a body field pulls its own record and + // no more than that. + const records = owing.length > 0 ? await this.batchGet(owing) : new Map>() + + const out = new Map>() + for (const id of ids) { + const fromIndex = served.get(id) + const record = records.get(id) + // An id neither the index nor storage knows is not a row. + if (fromIndex === undefined && record === undefined) continue + out.set(id, this.projectEntity(id, wanted, fromIndex, record)) + } + return out + } + + /** + * Build one projected entity: `id`, plus exactly the requested fields that + * something could supply. + * + * Values come from the index first and the record second, and they must agree + * — the index only reports what it can serve exactly, so a field it served is + * the record's value. A field neither has is omitted rather than set to + * `undefined`: absent and present-and-undefined are different answers, and a + * caller checking `'slug' in row.metadata` deserves the true one. + * + * @param id - The entity id, always present on the result. + * @param fields - The requested index keys. + * @param fromIndex - What the index served for this id, if anything. + * @param record - The canonical entity, if one had to be read. + * @returns The projected entity. + */ + private projectEntity( + id: string, + fields: readonly string[], + fromIndex: Record | undefined, + record: Entity | undefined + ): Entity { + const projected: Record = { id } + const metadata: Record = {} + let sawMetadata = false + + for (const field of fields) { + let value: unknown + let found = false + if (fromIndex !== undefined && field in fromIndex) { + value = fromIndex[field] + found = true + } else if (record !== undefined) { + if (field.startsWith('system.')) { + const inner = field.slice('system.'.length) + const bag = record as unknown as Record + if (inner in bag && bag[inner] !== undefined) { + value = bag[inner] + found = true + } + } else { + const bag = (record.metadata ?? {}) as Record + if (field in bag) { + value = bag[field] + found = true + } + } + } + if (!found) continue + + if (field.startsWith('system.')) { + projected[field.slice('system.'.length)] = value + } else { + metadata[field] = value + sawMetadata = true + } + } + + if (sawMetadata) projected.metadata = metadata + return projected as unknown as Entity + } + async batchGet(ids: string[], options?: GetOptions): Promise>> { // Canonical read (see get): resolves by id from storage, no derived index. await this.ensureInitialized({ needs: [] }) @@ -8037,7 +8211,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8074,7 +8248,7 @@ export class Brainy implements BrainyInterface { if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8102,7 +8276,7 @@ export class Brainy implements BrainyInterface { const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8337,7 +8511,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8365,7 +8539,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8470,6 +8644,28 @@ export class Brainy implements BrainyInterface { }) } + // PROJECTION TRIM — applied once, here, AFTER the integrity guard, so every + // find() path is trimmed uniformly and the guard still saw the evidence it + // needs. Hydration carried the guard's fields alongside the projection; + // this removes them, leaving exactly what the caller named. + // + // Rows that reached here from a path the seam does not hydrate (a vector or + // text leg builds its own entities) are trimmed from what they already + // hold, so the ANSWER is the same everywhere — only the cost differs, and + // only on the paths that still read a record. + if (params.fields !== undefined && params.fields.length > 0 && result.length > 0) { + const named = [...new Set(params.fields)] + result = result.map((r) => { + const projected = this.projectEntity( + r.id, + named, + undefined, + r.entity as unknown as Entity + ) + return { ...r, entity: projected } as typeof r + }) + } + // includeVectors — opt-in vector hydration. Default (false) keeps the perf // contract: every result path above builds entities via the metadata-only // fast path, so `entity.vector` is the empty stub. When requested, fetch the @@ -16842,7 +17038,7 @@ export class Brainy implements BrainyInterface { ordered = valued.map((v) => v.id) } const pageIds = ordered.slice(offset, offset + limit) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) const results: Result[] = [] for (const id of pageIds) { const entity = entitiesMap.get(id) diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 4fe45bff..48f4a963 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -292,6 +292,58 @@ export class ColumnStore implements ColumnStoreProvider { return result } + /** + * Read this column's value for each of `entityIntIds` — the per-id read + * behind `find({ fields })`. + * + * Every other read door here answers "which entities have this value". A + * projection asks the opposite — "what value does this entity have" — and + * without it a projection has to go to the canonical record for a field the + * column is already holding. + * + * The column is walked ONCE and the wanted ids are picked out as they pass, + * so the cost is O(column) per field rather than O(ids x column). Later + * sources win: the tail buffer holds writes newer than any segment, and + * within the segments a later one supersedes an earlier, exactly as `filter` + * treats them. + * + * Values are EXACT — this store keeps raw values, not the bucketed form the + * sparse index uses for range queries — which is what makes it safe to + * project from. Deleted entities are skipped; an id with no value in this + * column is simply absent from the result. + * + * @param field - Field name to read. + * @param entityIntIds - Entity integer ids to read values for. + * @returns `entityIntId -> value` for the ids this column holds. + */ + async valuesForIds( + field: string, + entityIntIds: Iterable + ): Promise> { + const wanted = new Set(entityIntIds) + const out = new Map() + if (wanted.size === 0 || !this.hasField(field)) return out + + const deleted = this.deletedEntities.get(field) + const take = (entry: { value: number | string; entityIntId: number }): void => { + if (!wanted.has(entry.entityIntId)) return + if (deleted && deleted.has(entry.entityIntId)) return + out.set(entry.entityIntId, entry.value) + } + + // Segments oldest -> newest, then the tail: a later write overwrites an + // earlier one for the same id. + const cursors = await this.getSegmentCursors(field) + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) take(entry) + } + const tailCursor = this.getTailBufferCursor(field) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) take(entry) + } + return out + } + /** * Range filter: find entities where field is within the bounds. * diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 4f4339f4..92e3057a 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2025-09-29T10:10:00-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/plugin.ts b/src/plugin.ts index 64abfe26..23a8c883 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -495,6 +495,45 @@ export interface MetadataIndexProvider { query: string, ids: readonly string[] ): Promise> + /** + * @description OPTIONAL: read named SCALAR fields for many ids at once, from + * the index's own value storage, WITHOUT touching the canonical record. + * + * This is the door behind `find/get/related({ fields })`. A list view that + * needs a title and a slug currently hydrates the whole record for every row + * — document bodies included — and then discards almost all of it. Serving + * the named scalars from the index turns that into an index read. + * + * ## The contract, and the one rule that makes it safe + * + * **Return only what you can serve EXACTLY, and say what you served.** The + * answer is a per-id map of the fields this index actually resolved; the + * caller diffs it against what was requested and reads the canonical record + * for the remainder. An implementation must therefore OMIT a field rather + * than approximate it — and omission costs only a record read, while a wrong + * value is a wrong answer nobody can see. + * + * That rule is not hypothetical. This engine's own index buckets + * `system.createdAt` and `system.updatedAt` to the minute for range queries, + * so it cannot serve them exactly and omits them. An engine whose column + * store holds raw values can serve the same fields — so the two answer + * differently in COST and identically in CONTENT, which is the only + * difference a projection door is allowed to have. + * + * A field absent from an entity is simply absent from that entity's map. It + * is never an error, and never a `null` standing in for one: absent and + * present-and-null are different answers. + * + * @param ids - Canonical entity ids to read. + * @param fields - Index KEYS (bare = user metadata, `system.*` = engine + * scalar), already address-resolved by the caller. + * @returns `id → { field: value }` for the fields this index served exactly. + * Ids with nothing to serve may be omitted entirely. + */ + getScalarsForIds?( + ids: readonly string[], + fields: readonly string[] + ): Promise>> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index a0d55c1e..b99f0261 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -561,6 +561,33 @@ export interface UpdateRelationParams { * refusal with the fix in hand beats a silent behavior flip. */ export interface FindParams { + /** + * **Field projection** — return only these fields on each row, instead of the + * whole record. + * + * A list view that shows a title and a slug does not need the document body, + * yet without a projection every row hydrates its full record and throws + * almost all of it away. Naming the fields lets them be served from the index + * itself: a scalar the index holds exactly is read from the index, and the + * canonical record is opened ONLY when a requested field cannot be. + * + * Field names follow the one addressing law: a bare name is the user's + * metadata (`'title'`), and `system.*` is an engine scalar + * (`'system.createdAt'`). + * + * - **Absent** ⇒ the full record, exactly as before. + * - A requested field the entity does not carry is simply **absent** from the + * row. It is never an error — a projection asks "give me these if you have + * them", so an optional field must not turn a list into a failure. + * - Every returned row carries `id` (and, on `find`, `score`) regardless: a + * row you cannot identify is not a row. + * + * @example + * // A list page: two user fields and one engine scalar, no document bodies. + * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 }) + */ + fields?: readonly string[] + // Vector Intelligence /** Natural language or semantic search query (embedded and matched via HNSW + text index) */ query?: string @@ -789,6 +816,12 @@ export interface SimilarParams { * Added string ID shorthand syntax */ export interface RelatedParams { + // NOTE: `fields` is deliberately NOT offered here. A Relation carries `from` + // and `to` as IDS and hydrates no entity record, so there is nothing for a + // projection to trim — the param would be decorative. Projecting the + // ENDPOINTS would be a new capability (related() hydrating entities), not a + // projection of an existing one, and it belongs in its own decision. + /** * Filter by source entity ID * @@ -1414,6 +1447,33 @@ export interface ImportResult { * */ export interface GetOptions { + /** + * **Field projection** — return only these fields on each row, instead of the + * whole record. + * + * A list view that shows a title and a slug does not need the document body, + * yet without a projection every row hydrates its full record and throws + * almost all of it away. Naming the fields lets them be served from the index + * itself: a scalar the index holds exactly is read from the index, and the + * canonical record is opened ONLY when a requested field cannot be. + * + * Field names follow the one addressing law: a bare name is the user's + * metadata (`'title'`), and `system.*` is an engine scalar + * (`'system.createdAt'`). + * + * - **Absent** ⇒ the full record, exactly as before. + * - A requested field the entity does not carry is simply **absent** from the + * row. It is never an error — a projection asks "give me these if you have + * them", so an optional field must not turn a list into a failure. + * - Every returned row carries `id` (and, on `find`, `score`) regardless: a + * row you cannot identify is not a row. + * + * @example + * // A list page: two user fields and one engine scalar, no document bodies. + * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 }) + */ + fields?: readonly string[] + /** * Include 384-dimensional vector embeddings in the response * diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a3aa7679..d07fa8d7 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2805,6 +2805,67 @@ export class MetadataIndexManager implements MetadataIndexProvider { return order === 'asc' ? comparison : -comparison } + /** + * Read named scalar fields for many ids from the COLUMN STORE, without + * touching the canonical record — the `find({ fields })` door. + * + * ## Why the column store and not the sparse index + * + * The column store keeps RAW values; the sparse index keeps a normalized, + * bucketed form built for range queries — `system.createdAt` is indexed at + * minute precision there. A projection served from the sparse index would + * hand back a value that differs from the record's, which is a wrong answer + * nobody can see. So this door reads the column store, and a field the + * column store does not hold is OMITTED rather than approximated. + * + * ## Why batched + * + * `getFieldValueForEntity` answers one (id, field) pair by walking the + * field's storage; called per row it re-walks the same column for every id. + * This walks each column ONCE and picks out every requested id as it passes: + * O(fields x column) instead of O(ids x fields x column). + * + * Omission is always safe — it costs the caller a record read. The caller + * diffs what it asked for against what came back and reads records for the + * remainder, so an index that can serve nothing is slow, never wrong. + * + * @param ids - Canonical entity ids. + * @param fields - Index keys (bare = user metadata, `system.*` = engine scalar). + * @returns `id -> { field: value }` for exactly the pairs this index served. + */ + async getScalarsForIds( + ids: readonly string[], + fields: readonly string[] + ): Promise>> { + const out = new Map>() + if (ids.length === 0 || fields.length === 0) return out + + // int -> id, so a column hit resolves back to the caller's id. An id the + // mapper does not know cannot be in any column, so it is simply absent. + const idByInt = new Map() + for (const id of ids) { + const intId = this.idMapper.getInt(id) + if (intId !== undefined) idByInt.set(intId, id) + } + if (idByInt.size === 0) return out + + for (const field of fields) { + if (!this.columnStore.hasField(field)) continue + const values = await this.columnStore.valuesForIds(field, idByInt.keys()) + for (const [intId, value] of values) { + const id = idByInt.get(intId) + if (id === undefined) continue + let row = out.get(id) + if (row === undefined) { + row = {} + out.set(id, row) + } + row[field] = value + } + } + return out + } + async getFieldValueForEntity(entityId: string, field: string): Promise { // `field` arrives as a FROZEN INDEX KEY (bare = user metadata; // 'system.' = engine scalar). Storage fallbacks read the matching diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts new file mode 100644 index 00000000..3718dc4f --- /dev/null +++ b/tests/integration/find-fields-projection.test.ts @@ -0,0 +1,224 @@ +/** + * @module tests/integration/find-fields-projection + * @description **Field projection** — `find/get({ fields })` returns only the + * named fields, and serves them from the index when it can. + * + * A list view that shows a title and a slug does not need the document body, + * yet without a projection every row hydrates its whole record and discards + * almost all of it. These pins hold the two halves of the fix: + * + * **The answer.** A projected row is a SUBSET of the full row — for every + * requested field, the projected value equals the value the same query returns + * unprojected. Absent `fields` is byte-identical to today. A requested field the + * entity does not carry is simply absent, never an error. `system.*` resolves to + * the engine scalar, a bare name to the user's metadata. + * + * **The cost.** When every requested field is index-served, the canonical + * record is never opened — asserted by counting reads, not by timing them, so + * it cannot flake into a false green. When one requested field is NOT + * index-served (a body field, or a bucketed timestamp), exactly the owing rows + * are read and the rest are still served from the index. + */ +import { describe, it, expect, beforeAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' +import { generateTestVector } from '../helpers/test-factory' + +/** Rows carrying a title, a slug, and a large body nobody wants in a list. */ +const ROWS = 12 +const BODY = 'x'.repeat(4096) + +describe('find/get({ fields }) — projection', () => { + let brain: Brainy + const ids: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + for (let i = 0; i < ROWS; i++) { + ids.push( + await brain.add({ + id: `post-${i}`, + data: `post ${i}`, + type: NounType.Thing, + metadata: { + kind: 'post', + title: `Title ${i}`, + slug: `slug-${i}`, + rank: i, + body: BODY, + // Only some rows carry this, so "missing is absent" is exercised + // by real data rather than by a name nothing ever had. + ...(i % 2 === 0 ? { featured: true } : {}) + }, + vector: generateTestVector() + }) + ) + } + // Persist so the column store holds the values a projection reads from. + await brain.flush() + }) + + /** Count canonical record reads for one call. */ + const countingReads = async (body: () => Promise): Promise<{ out: R; reads: number }> => { + const spy = vi.spyOn(brain as any, 'batchGet') + try { + const out = await body() + const reads = spy.mock.calls.reduce( + (n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0), + 0 + ) + return { out, reads } + } finally { + spy.mockRestore() + } + } + + it('absent fields is byte-identical to today', async () => { + const params = { where: { kind: 'post' }, limit: 5 } + const a = await brain.find({ ...params }) + const b = await brain.find({ ...params, fields: undefined }) + expect(JSON.stringify(b)).toBe(JSON.stringify(a)) + }) + + it('a projected row is a SUBSET of the full row, field for field', async () => { + const shapes: Array> = [ + { where: { kind: 'post' }, limit: 6 }, + { where: { kind: 'post' }, limit: 6, offset: 3 }, + { where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 }, + { where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 } + ] + for (const shape of shapes) { + const full = await brain.find(shape as never) + const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never) + expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id)) + for (let i = 0; i < full.length; i++) { + const fullMeta = (full[i].entity.metadata ?? {}) as Record + const projMeta = (projected[i].entity.metadata ?? {}) as Record + expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title) + expect(projMeta.slug).toEqual(fullMeta.slug) + } + } + }) + + it('returns ONLY the named fields — the body never rides along', async () => { + const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 }) + expect(rows).toHaveLength(4) + for (const r of rows) { + const meta = (r.entity.metadata ?? {}) as Record + expect(Object.keys(meta)).toEqual(['title']) + expect(meta.body).toBeUndefined() + // Identity always survives a projection: a row you cannot identify is + // not a row. + expect(typeof r.id).toBe('string') + expect(r.entity.id).toBe(r.id) + } + }) + + it('a missing field is simply ABSENT — never an error', async () => { + // `featured` exists on half the rows; `no-such-field` on none. Neither + // throws, and neither appears as an explicit undefined. + const rows = await brain.find({ + where: { kind: 'post' }, + fields: ['title', 'featured', 'no-such-field'], + limit: ROWS + }) + expect(rows.length).toBeGreaterThan(0) + let withFeatured = 0 + for (const r of rows) { + const meta = (r.entity.metadata ?? {}) as Record + expect('no-such-field' in meta).toBe(false) + if ('featured' in meta) withFeatured += 1 + } + // Real data, not a name nothing ever had: some rows carry it, some do not. + expect(withFeatured).toBeGreaterThan(0) + expect(withFeatured).toBeLessThan(rows.length) + }) + + it('a strict address resolver is NOT on this path', async () => { + // orderBy throws UnresolvableFieldError for an unknown user key, because a + // typo there silently changes the order. A projection must not inherit that + // strictness: the honest answer to "give me this if you have it" is silence. + await expect( + brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 }) + ).resolves.toBeInstanceOf(Array) + }) + + it('system.* resolves to the engine scalar, a bare name to user metadata', async () => { + const full = await brain.find({ where: { kind: 'post' }, limit: 3 }) + const rows = await brain.find({ + where: { kind: 'post' }, + fields: ['system.createdAt', 'title'], + limit: 3 + }) + for (let i = 0; i < rows.length; i++) { + expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt) + const meta = (rows[i].entity.metadata ?? {}) as Record + expect(meta.title).toEqual((full[i].entity.metadata as any).title) + // The engine scalar lands at the top level, not in the metadata bag — + // the two address spaces never shadow each other. + expect('system.createdAt' in meta).toBe(false) + expect('createdAt' in meta).toBe(false) + } + }) + + it('reads NO canonical record when every requested field is index-served', async () => { + // The cost pin, counted rather than timed. `title` and `slug` are ordinary + // indexed user fields, so the index can serve them exactly. + const { out, reads } = await countingReads(() => + brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS }) + ) + expect(out.length).toBeGreaterThan(0) + expect(reads).toBe(0) + }) + + it('reads records only for the rows that owe an un-served field', async () => { + // `body` is not a scalar the index serves, so the record must be opened — + // but the projection still returns only the named fields. + const { out, reads } = await countingReads(() => + brain.find({ where: { kind: 'post' }, fields: ['title', 'body'], limit: 4 }) + ) + expect(out).toHaveLength(4) + expect(reads).toBe(4) + for (const r of out) { + const meta = (r.entity.metadata ?? {}) as Record + expect(meta.body).toBe(BODY) + expect(Object.keys(meta).sort()).toEqual(['body', 'title']) + } + }) + + it('get({ fields }) projects a single row through the same seam', async () => { + const full = await brain.get(ids[0]) + const projected = await brain.get(ids[0], { fields: ['title', 'slug'] }) + expect(projected).not.toBeNull() + expect(projected!.id).toBe(full!.id) + const fullMeta = (full!.metadata ?? {}) as Record + const projMeta = (projected!.metadata ?? {}) as Record + expect(projMeta.title).toEqual(fullMeta.title) + expect(projMeta.slug).toEqual(fullMeta.slug) + expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title']) + expect((projected as any).body).toBeUndefined() + }) + + it('get({ fields }) reads no record when the index serves the fields', async () => { + const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] })) + expect(reads).toBe(0) + }) + + it('the provider door serves only what it can serve EXACTLY', async () => { + // The bucketed timestamps are indexed at minute precision for range + // queries. The door must omit them rather than hand back a bucket that + // differs from the record — omission costs a read, a wrong value is a wrong + // answer nobody can see. + const index = (brain as any).metadataIndex + const served = await index.getScalarsForIds(ids.slice(0, 3), [ + 'title', + 'system.createdAt' + ]) + expect(served.size).toBeGreaterThan(0) + for (const [, row] of served) { + expect('title' in row).toBe(true) + expect('system.createdAt' in row).toBe(false) + } + }) +})