feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal
Some checks failed
CI / Node 22 (push) Successful in 12m15s
CI / Node 24 (push) Has been cancelled
CI / Bun (latest) (push) Has been cancelled

A first adopter's namespace migration went 341 red on one class: the
never-carried-field refusal firing on CORRECT filters against fresh and
sparse stores — a freshly provisioned tenant refused its own first
filtered read, with the did-you-mean built for typos firing hardest on
day-one stores where nothing is wrong.

The ruled cut: a WHERE filter naming a field no row carries is SERVED
OPERATOR-TRUTHFULLY — eq/in/range/contains answer [] (nothing carries
it, nothing matches); ne and exists:false answer ALL rows (the equally
true complement — a blanket empty here would be silently wrong, which is
why the simpler cut was rejected); exists:true answers []. Served from
the field registry, with the did-you-mean demoted to a once-per-field
WARN. orderBy and genuinely ambiguous addresses KEEP their hard typed
refusals: no truthful order exists over an uncarried field, and
ambiguity is a contract error while absence is data.

Mechanics: the negative operator absorbs the FIELD_NOT_INDEXED throw as
its empty exclude set (the clause-level catch correctly zeroes positive
operators only); the egress matcher already agreed. Plus the
provider-seam belt: a field refusal thrown by a replacement metadata
manager is normalized to THIS package's UnresolvableFieldError at every
filter call site — one class identity for consumers, instanceof works
(a first adopter's cross-package finding).

Conformance: tests/conformance/sparse-store-cut.test.ts — the shared
operator rows both engines run (positive-empty, negative-all,
fresh-tenant day-one, orderBy refusal kept, compound composition).
Gates: unit 2065/2065 · integration 832 · conformance 36/36.
This commit is contained in:
David Snelling 2026-08-12 15:57:19 -07:00
parent df96fccfd1
commit 7b67db4d0c
5 changed files with 188 additions and 8 deletions

View file

@ -197,6 +197,7 @@ import { GenerationConflictError, StoreInconsistentError } from './db/errors.js'
import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js'
import { assessIndexReadiness } from './utils/indexReadiness.js'
import { reconstructNounWrapper } from './db/factLog.js'
import { asBrainyFieldRefusal } from './db/fieldAddressing.js'
import {
readLogAuthority,
runLogCompletenessOracle,
@ -4107,7 +4108,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const probeServes = async (): Promise<boolean> => {
try {
const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value })
const ids = await this.filterIdsBelted({ [p.field]: p.value })
return ids.includes(p.id)
} catch {
// FIELD_NOT_INDEXED for a field a persisted entity actually holds is
@ -6447,7 +6448,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// 'visibility' key would address the USER's metadata bag under the
// field-addressing law and silently hide nothing (VFS/system entities
// would leak into every default read).
const ids = await this.metadataIndex.getIdsForFilter({
const ids = await this.filterIdsBelted({
'system.visibility': excluded.length === 1 ? excluded[0] : { oneOf: excluded }
})
return new Set(ids)
@ -6655,7 +6656,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// offset stays 0 because the visibility filter + slice happen here. The JS
// index ignores the bound and returns all matches (behaviour unchanged).
const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size
filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 })
filteredIds = await this.filterIdsBelted(filter, { limit: pageEnd, offset: 0 })
}
// Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact.
@ -6727,7 +6728,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// filter returns nothing from getIdsForFilter, so the unfiltered case below uses
// getNouns instead (it returns all nouns, including their visibility).
if (Object.keys(filter).length > 0) {
let filteredIds = await this.metadataIndex.getIdsForFilter(filter)
let filteredIds = await this.filterIdsBelted(filter)
// Visibility hard filter — drop hidden ids BEFORE pagination.
if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))
const pageIds = filteredIds.slice(offset, offset + limit)
@ -6778,7 +6779,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
preResolvedFilter = this.buildMetadataFilter(params)
preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter)
preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)
// Visibility hard filter — restrict the HNSW candidate set to non-hidden ids.
if (hiddenIds.size > 0) {
@ -11548,6 +11549,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`)
* ```
*/
/**
* The provider-seam belt for filter reads: whatever manager serves
* getIdsForFilter (the JS twin or a native replacement), a field refusal
* crossing this seam is normalized to BRAINY'S UnresolvableFieldError
* one class identity for consumers, never a foreign twin that fails
* instanceof. All other errors pass through untouched.
*/
private async filterIdsBelted(
filter: unknown,
opts?: { limit?: number; offset?: number }
): Promise<string[]> {
try {
return await this.metadataIndex.getIdsForFilter(filter, opts)
} catch (err) {
const normalized = asBrainyFieldRefusal(err)
if (normalized) throw normalized
throw err
}
}
async getIndexStatus(): Promise<{
initialized: boolean
lazyRebuildCompleted: boolean
@ -12145,7 +12167,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
const filteredIds = await this.metadataIndex.getIdsForFilter(filter)
const filteredIds = await this.filterIdsBelted(filter)
return filteredIds.length
}
@ -12217,7 +12239,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
const filteredIds = await this.metadataIndex.getIdsForFilter(filterObj)
const filteredIds = await this.filterIdsBelted(filterObj)
// Stream filtered entities in batches for memory efficiency
const batchSize = 100