From 7b67db4d0c2f89468ea397ddd57c67dee380db9c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 12 Aug 2026 15:57:19 -0700 Subject: [PATCH] =?UTF-8?q?feat(query):=20the=20sparse-store=20cut=20?= =?UTF-8?q?=E2=80=94=20where=20on=20a=20never-carried=20field=20serves=20o?= =?UTF-8?q?perator=20truth,=20never=20a=20refusal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A first adopter's namespace migration went 341 red on one class: the never-carried-field refusal firing on CORRECT filters against fresh and sparse stores — a freshly provisioned tenant refused its own first filtered read, with the did-you-mean built for typos firing hardest on day-one stores where nothing is wrong. The ruled cut: a WHERE filter naming a field no row carries is SERVED OPERATOR-TRUTHFULLY — eq/in/range/contains answer [] (nothing carries it, nothing matches); ne and exists:false answer ALL rows (the equally true complement — a blanket empty here would be silently wrong, which is why the simpler cut was rejected); exists:true answers []. Served from the field registry, with the did-you-mean demoted to a once-per-field WARN. orderBy and genuinely ambiguous addresses KEEP their hard typed refusals: no truthful order exists over an uncarried field, and ambiguity is a contract error while absence is data. Mechanics: the negative operator absorbs the FIELD_NOT_INDEXED throw as its empty exclude set (the clause-level catch correctly zeroes positive operators only); the egress matcher already agreed. Plus the provider-seam belt: a field refusal thrown by a replacement metadata manager is normalized to THIS package's UnresolvableFieldError at every filter call site — one class identity for consumers, instanceof works (a first adopter's cross-package finding). Conformance: tests/conformance/sparse-store-cut.test.ts — the shared operator rows both engines run (positive-empty, negative-all, fresh-tenant day-one, orderBy refusal kept, compound composition). Gates: unit 2065/2065 · integration 832 · conformance 36/36. --- src/brainy.ts | 36 +++++++-- src/db/fieldAddressing.ts | 22 ++++++ src/utils/metadataIndex.ts | 53 ++++++++++++- tests/conformance/sparse-store-cut.test.ts | 82 ++++++++++++++++++++ tests/unit/test-suite-coverage-guard.test.ts | 3 + 5 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 tests/conformance/sparse-store-cut.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index a0f06931..785d10e5 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -197,6 +197,7 @@ import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' import { assessIndexReadiness } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' +import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { readLogAuthority, runLogCompletenessOracle, @@ -4107,7 +4108,7 @@ export class Brainy implements BrainyInterface { const probeServes = async (): Promise => { try { - const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value }) + const ids = await this.filterIdsBelted({ [p.field]: p.value }) return ids.includes(p.id) } catch { // FIELD_NOT_INDEXED for a field a persisted entity actually holds is @@ -6447,7 +6448,7 @@ export class Brainy implements BrainyInterface { // 'visibility' key would address the USER's metadata bag under the // field-addressing law and silently hide nothing (VFS/system entities // would leak into every default read). - const ids = await this.metadataIndex.getIdsForFilter({ + const ids = await this.filterIdsBelted({ 'system.visibility': excluded.length === 1 ? excluded[0] : { oneOf: excluded } }) return new Set(ids) @@ -6655,7 +6656,7 @@ export class Brainy implements BrainyInterface { // offset stays 0 because the visibility filter + slice happen here. The JS // index ignores the bound and returns all matches (behaviour unchanged). const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size - filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 }) + filteredIds = await this.filterIdsBelted(filter, { limit: pageEnd, offset: 0 }) } // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. @@ -6727,7 +6728,7 @@ export class Brainy implements BrainyInterface { // filter returns nothing from getIdsForFilter, so the unfiltered case below uses // getNouns instead (it returns all nouns, including their visibility). if (Object.keys(filter).length > 0) { - let filteredIds = await this.metadataIndex.getIdsForFilter(filter) + let filteredIds = await this.filterIdsBelted(filter) // Visibility hard filter — drop hidden ids BEFORE pagination. if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) const pageIds = filteredIds.slice(offset, offset + limit) @@ -6778,7 +6779,7 @@ export class Brainy implements BrainyInterface { if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { preResolvedFilter = this.buildMetadataFilter(params) - preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) + preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter) // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. if (hiddenIds.size > 0) { @@ -11548,6 +11549,27 @@ export class Brainy implements BrainyInterface { * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) * ``` */ + + /** + * The provider-seam belt for filter reads: whatever manager serves + * getIdsForFilter (the JS twin or a native replacement), a field refusal + * crossing this seam is normalized to BRAINY'S UnresolvableFieldError — + * one class identity for consumers, never a foreign twin that fails + * instanceof. All other errors pass through untouched. + */ + private async filterIdsBelted( + filter: unknown, + opts?: { limit?: number; offset?: number } + ): Promise { + try { + return await this.metadataIndex.getIdsForFilter(filter, opts) + } catch (err) { + const normalized = asBrainyFieldRefusal(err) + if (normalized) throw normalized + throw err + } + } + async getIndexStatus(): Promise<{ initialized: boolean lazyRebuildCompleted: boolean @@ -12145,7 +12167,7 @@ export class Brainy implements BrainyInterface { } } - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + const filteredIds = await this.filterIdsBelted(filter) return filteredIds.length } @@ -12217,7 +12239,7 @@ export class Brainy implements BrainyInterface { } } - const filteredIds = await this.metadataIndex.getIdsForFilter(filterObj) + const filteredIds = await this.filterIdsBelted(filterObj) // Stream filtered entities in batches for memory efficiency const batchSize = 100 diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index 21689319..da04e74c 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -261,6 +261,28 @@ export function buildUnresolvableMessage( * the fix ships inside the error. Thrown by the query layer with index * knowledge, never by the pure parser. */ +/** + * Cross-package identity normalizer (the seam belt): the native accelerator + * throws ITS OWN UnresolvableFieldError class, which fails `instanceof` + * against this package's export — consumers were forced to match by name. + * Every provider-boundary catch routes suspected field-refusals through + * here: a foreign refusal (matched by name, duck fields tolerated) is + * rethrown as THIS package's class, so exactly one identity ever reaches + * consumers. Anything else returns null (caller rethrows the original). + */ +export function asBrainyFieldRefusal(err: unknown): UnresolvableFieldError | null { + if (err instanceof UnresolvableFieldError) return err + const e = err as { name?: string; message?: string; raw?: string; kind?: string } | null + if (e && e.name === 'UnresolvableFieldError') { + return new UnresolvableFieldError( + e.raw ?? 'unknown-field', + (e.kind as FieldAddressKind) ?? 'entity', + e.message + ) + } + return null +} + export class UnresolvableFieldError extends Error { public readonly raw: string public readonly kind: FieldAddressKind diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 894f3fd3..13cf3bb4 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1908,6 +1908,35 @@ export class MetadataIndexManager implements MetadataIndexProvider { * index (early-stop at `offset+limit`); the JS index returns ALL matches and lets * the caller window them, so `_opts` is intentionally ignored here. */ + /** Once-per-field throttle for the sparse-store did-you-mean WARN. */ + private readonly warnedNeverCarried = new Set() + + /** + * THE SPARSE-STORE CUT (ruled 2026-08-12): a WHERE filter naming a field + * no row carries is SERVED OPERATOR-TRUTHFULLY (eq/range/contains → []; + * ne/exists:false → all rows; exists:true → []) — the JS evaluator below + * already computes exactly these truths via complements — with the + * did-you-mean demoted to this throttled WARN. A fresh store's first + * filtered read is a correct empty answer, never a refusal. orderBy and + * ambiguous addresses KEEP their hard refusals (no truthful order + * exists; ambiguity is a contract error — absence is data). + */ + /** Is this field known to the index at all (any row ever carried it)? */ + private fieldRegistryHas(field: string): boolean { + return this.fieldStats.has(field) + } + + private warnNeverCarriedOnce(field: string): void { + if (this.warnedNeverCarried.has(field)) return + this.warnedNeverCarried.add(field) + prodLog.warn( + `[MetadataIndex] filter names field '${field}' which no row carries — ` + + `serving the operator-truthful answer (empty for positive matches; ` + + `the complement for ne/exists:false). If this is a typo, check the ` + + `field name; refusals remain on orderBy.` + ) + } + async getIdsForFilter(filter: any, _opts?: { limit?: number; offset?: number }): Promise { if (!filter || Object.keys(filter).length === 0) { return [] @@ -1984,6 +2013,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { const address = parseFieldAddress(rawField, 'entity') const field = address.scope === 'system' ? `system.${address.field}` : address.field + // Sparse-store cut: a user field no row carries serves operator-truth + // below (the evaluators' complements are already correct) — announce + // it once so a typo is findable without breaking a fresh store. + if ( + address.scope !== 'system' && + !(this.columnStore && this.columnStore.hasField(field)) && + !this.fieldRegistryHas(field) + ) { + this.warnNeverCarriedOnce(field) + } + let fieldResults: string[] = [] try { @@ -2022,7 +2062,18 @@ export class MetadataIndexManager implements MetadataIndexProvider { // complement as a bitmap difference over the int-id universe rather // than materializing the whole corpus as UUID strings to filter it. const excludeInts: number[] = [] - for (const uuid of await this.getIds(field, operand)) { + // Sparse-store truth: a never-carried field has NOTHING to + // exclude — the complement of nothing is EVERYTHING. getIds + // throws FIELD_NOT_INDEXED there; the clause-level catch + // would wrongly zero this NEGATIVE operator, so absorb it + // here as the empty exclude set (the ruled operator-truth). + let neMatches: string[] = [] + try { + neMatches = await this.getIds(field, operand) + } catch { + neMatches = [] + } + for (const uuid of neMatches) { const intId = this.idMapper.getInt(uuid) if (intId !== undefined) excludeInts.push(intId) } diff --git a/tests/conformance/sparse-store-cut.test.ts b/tests/conformance/sparse-store-cut.test.ts new file mode 100644 index 00000000..8a06c98c --- /dev/null +++ b/tests/conformance/sparse-store-cut.test.ts @@ -0,0 +1,82 @@ +/** + * @module tests/conformance/sparse-store-cut + * @description THE SPARSE-STORE CUT (ruled 2026-08-12) — the shared + * conformance rows both engines run: a WHERE filter naming a field NO row + * carries is SERVED OPERATOR-TRUTHFULLY, never refused: + * eq / in / range / contains → [] (nothing carries it → nothing matches) + * ne / exists:false → ALL rows (equally true — blanket-empty here + * would be the outlawed silent wrong) + * exists:true → [] + * orderBy on an unresolvable field KEEPS the hard refusal (no truthful + * order exists). The did-you-mean demotes to a throttled WARN on the serve. + * A fresh tenant's first filtered read is a correct empty answer — the + * 341-red first-adopter class, closed. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy, UnresolvableFieldError } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) +}) + +async function corpus(): Promise<{ brain: Brainy; ids: string[] }> { + const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) + await b.init() + brains.push(b) + const ids: string[] = [] + for (let i = 0; i < 4; i++) { + ids.push( + await b.add({ data: `row ${i}`, type: NounType.Document, metadata: { carried: i } }) + ) + } + return { brain: b, ids } +} + +describe('sparse-store cut — operator-truthful serve on never-carried fields', () => { + it('positive matches serve EMPTY: eq, in, range, contains', async () => { + const { brain } = await corpus() + expect(await brain.find({ where: { ghost: 'x' }, limit: 10 })).toEqual([]) + expect(await brain.find({ where: { ghost: { in: ['a', 'b'] } }, limit: 10 })).toEqual([]) + expect(await brain.find({ where: { ghost: { gt: 5 } }, limit: 10 })).toEqual([]) + expect(await brain.find({ where: { ghost: { exists: true } }, limit: 10 })).toEqual([]) + }) + + it('negative matches serve ALL rows: ne and exists:false (the truth, not blanket-empty)', async () => { + const { brain, ids } = await corpus() + const ne = await brain.find({ where: { ghost: { ne: 'x' } }, limit: 10 }) + expect(ne.map((r) => r.id).sort()).toEqual([...ids].sort()) + const absent = await brain.find({ where: { ghost: { exists: false } }, limit: 10 }) + expect(absent.map((r) => r.id).sort()).toEqual([...ids].sort()) + }) + + it('the fresh-tenant day-one shape: an EMPTY store answers its first filtered read with [], never a refusal', async () => { + const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) + await b.init() + brains.push(b) + expect(await b.find({ where: { status: 'open' }, limit: 50 })).toEqual([]) + expect(await b.find({ where: { date: { gte: '2026-01-01' } }, limit: 50 })).toEqual([]) + }) + + it('orderBy on an unresolvable field KEEPS the typed refusal', async () => { + const { brain } = await corpus() + await expect( + brain.find({ where: { carried: { gte: 0 } }, orderBy: 'system.notAScalar', limit: 10 }) + ).rejects.toThrow(UnresolvableFieldError) + }) + + it('compound filters: the never-carried clause composes truthfully with carried clauses', async () => { + const { brain, ids } = await corpus() + // carried>=2 AND ghost ne 'x' → the carried>=2 rows (ne-clause = all). + const both = await brain.find({ + where: { carried: { gte: 2 }, ghost: { ne: 'x' } }, + limit: 10 + }) + expect(both.map((r) => r.id).sort()).toEqual([ids[2], ids[3]].sort()) + // carried>=2 AND ghost eq 'x' → [] (eq-clause empties the intersection). + expect( + await brain.find({ where: { carried: { gte: 2 }, ghost: 'x' }, limit: 10 }) + ).toEqual([]) + }) +}) diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index 4b078146..c43b2cf2 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -37,6 +37,9 @@ const MANUAL_ONLY = new Set([ // (byte + fold digests) — runs in the explicit conformance gate stage, // same invocation family as the other conformance suites. 'tests/conformance/golden-log-fold.test.ts', + // The sparse-store cut's shared operator rows (both engines run these): + // explicit conformance-gate invocation, like its siblings. + 'tests/conformance/sparse-store-cut.test.ts', 'tests/api/performance-benchmarks.test.ts', 'tests/critical-neural-validation.test.ts', 'tests/critical-performance-benchmark.test.ts',