fix(metadata): the indexable-array bound is 256 — a keyword list is not a vector

64 cleared tags, authors and labels, but not the shape that actually turns up
in production metadata: a long keyword or participant list. 256 clears those
and still refuses every embedding this engine will ever meet — the narrowest
model it ships is 384-dimensional, so the two populations still do not overlap
and nobody has to tune anything. A vector parked in metadata throws by name;
a 200-keyword list writes and indexes.

The number lives in ONE place, `MAX_INDEXED_ARRAY_LENGTH`, and every message,
warning and pin derives it from there. Two pins still carried a literal:
metadata-vector-exclusion refused an array of exactly 100 — which sits UNDER
the new bound, so the case would have asserted a refusal that no longer
happens — and the array-bound suite named "all 64 elements" in a title and
picked its middle element as a hardcoded 't31'. Both derive from the constant
now, so the pins follow it wherever it goes rather than silently inverting the
next time it moves.
This commit is contained in:
David Snelling 2026-09-02 15:41:11 -07:00
parent a2820e81af
commit e435da787d
3 changed files with 19 additions and 11 deletions

View file

@ -412,18 +412,23 @@ export class MigrationInProgressError extends BrainyError {
* embedding parked in the metadata bag would mint 384 postings for one row. * embedding parked in the metadata bag would mint 384 postings for one row.
* The bound exists to keep that out of the index. * The bound exists to keep that out of the index.
* *
* 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above * 256 is hardcoded on purpose (the zero-config law: no knob). It sits far above
* every legitimate multi-value field the engine has seen tags, authors, * every legitimate multi-value field the engine has seen tags, authors,
* categories, labels, participant lists and far below any real embedding * categories, labels, keyword lists, participant lists and still below the
* width, so the two populations do not overlap and no caller has to tune it. * narrowest embedding this engine will ever meet (384 dimensions, the smallest
* model it ships), so the two populations do not overlap and no caller has to
* tune it. A vector parked in metadata is refused; a long keyword list is not.
* *
* It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array * It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array
* held eleven entries had that field skipped entirely and dropped out of every * held eleven entries had that field skipped entirely and dropped out of every
* filtered search on it, with no error, no warning and no way to tell the * filtered search on it, with no error, no warning and no way to tell the
* difference from "no row matches". A rule this consequential is a law with a * difference from "no row matches". A rule this consequential is a law with a
* name and a refusal, not a `continue`. * name and a refusal, not a `continue`.
*
* This is the ONE place the number lives. Every message, warning, doc line and
* pin derives it from here never a literal.
*/ */
export const MAX_INDEXED_ARRAY_LENGTH = 64 export const MAX_INDEXED_ARRAY_LENGTH = 256
/** /**
* A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}. * A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}.

View file

@ -161,7 +161,8 @@ describe('Metadata Vector Exclusion Fix', () => {
// silence at a bound of 10 — the field simply vanished from the index and // silence at a bound of 10 — the field simply vanished from the index and
// the row dropped out of every `where` on it, indistinguishably from "no // the row dropped out of every `where` on it, indistinguishably from "no
// row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES. // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES.
const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`) const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1
const largeArray = Array.from({ length: overTheBound }, (_, i) => `item${i}`)
const err = await brainy const err = await brainy
.add({ .add({
@ -176,7 +177,7 @@ describe('Metadata Vector Exclusion Fix', () => {
expect(err).toBeInstanceOf(MetadataArrayTooLargeError) expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
expect(err.field).toBe('items') expect(err.field).toBe('items')
expect(err.length).toBe(100) expect(err.length).toBe(overTheBound)
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
// Nothing was indexed from the refused write — no 'items' field, and above // Nothing was indexed from the refused write — no 'items' field, and above

View file

@ -15,9 +15,10 @@
* and the caller had no way to tell that from "no row matches". Eleven tags is * and the caller had no way to tell that from "no row matches". Eleven tags is
* not an exotic shape; the eleventh tag made the row invisible. * not an exotic shape; the eleventh tag made the row invisible.
* *
* THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64, * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH},
* hardcoded (the zero-config law: no knob), which clears every legitimate * hardcoded (the zero-config law: no knob), which clears every legitimate
* multi-value field and stays far below any embedding width. Above it the WRITE * multi-value field tags, authors, keyword lists and stays below the
* narrowest embedding this engine meets (384 dimensions). Above it the WRITE
* IS REFUSED by name `MetadataArrayTooLargeError`, carrying the field, the * IS REFUSED by name `MetadataArrayTooLargeError`, carrying the field, the
* length and the bound at `add`, `update`, `relate` and `updateRelation` * length and the bound at `add`, `update`, `relate` and `updateRelation`
* alike. Nothing is skipped in silence. * alike. Nothing is skipped in silence.
@ -65,7 +66,7 @@ describe('the indexable-array bound', () => {
} }
}) })
it('indexes right up to the bound — all 64 elements', async () => { it('indexes right up to the bound — every element of it', async () => {
await brain.add({ await brain.add({
id: 'at-bound', id: 'at-bound',
data: 'a row at the bound', data: 'a row at the bound',
@ -74,8 +75,9 @@ describe('the indexable-array bound', () => {
vector: [] vector: []
}) })
// The first, the last, and one in the middle. // The first, the last, and one in the middle — all derived from the
for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) { // bound, so the case follows the constant wherever it moves.
for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, `t${Math.floor(MAX_INDEXED_ARRAY_LENGTH / 2)}`]) {
const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound')) expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound'))
} }