David Snelling
e435da787d
64 cleared tags, authors and labels, but not the shape that actually turns up in production metadata: a long keyword or participant list. 256 clears those and still refuses every embedding this engine will ever meet — the narrowest model it ships is 384-dimensional, so the two populations still do not overlap and nobody has to tune anything. A vector parked in metadata throws by name; a 200-keyword list writes and indexes. The number lives in ONE place, `MAX_INDEXED_ARRAY_LENGTH`, and every message, warning and pin derives it from there. Two pins still carried a literal: metadata-vector-exclusion refused an array of exactly 100 — which sits UNDER the new bound, so the case would have asserted a refusal that no longer happens — and the array-bound suite named "all 64 elements" in a title and picked its middle element as a hardcoded 't31'. Both derive from the constant now, so the pins follow it wherever it goes rather than silently inverting the next time it moves.
244 lines
9.7 KiB
TypeScript
244 lines
9.7 KiB
TypeScript
/**
|
|
* @module tests/unit/utils/metadataIndex-array-bound
|
|
* @description THE INDEXABLE-ARRAY BOUND — a law with a name and a refusal,
|
|
* not a `continue`.
|
|
*
|
|
* THE DEFECT. An array-valued metadata field indexes one posting per element,
|
|
* so the index has always carried a ceiling. It was 10, and it was applied by a
|
|
* bare `continue` deep inside field extraction:
|
|
*
|
|
* if (Array.isArray(value) && value.length > 10) continue
|
|
*
|
|
* A row whose `tags` array held ELEVEN entries therefore had that field skipped
|
|
* entirely — no posting, no error, no warning. The row then failed to match
|
|
* every filtered search on `tags`, including `{ tags: 'a-tag-it-really-has' }`,
|
|
* and the caller had no way to tell that from "no row matches". Eleven tags is
|
|
* not an exotic shape; the eleventh tag made the row invisible.
|
|
*
|
|
* THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH},
|
|
* hardcoded (the zero-config law: no knob), which clears every legitimate
|
|
* multi-value field — tags, authors, keyword lists — and stays below the
|
|
* narrowest embedding this engine meets (384 dimensions). Above it the WRITE
|
|
* IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the
|
|
* length and the bound — at `add`, `update`, `relate` and `updateRelation`
|
|
* alike. Nothing is skipped in silence.
|
|
*
|
|
* THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an
|
|
* older engine under the old rule and read back by a rebuild, a catch-up fold
|
|
* or a remove. Refusing there would make an existing store un-rebuildable — so
|
|
* the row is admitted and the skipped field is NARRATED. Both sides are pinned.
|
|
*/
|
|
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'
|
|
import { Brainy } from '../../../src/brainy'
|
|
import { NounType, VerbType } from '../../../src/types/graphTypes'
|
|
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError'
|
|
import { resolveEntityId } from '../../../src/utils/idNormalization'
|
|
import { prodLog } from '../../../src/utils/logger'
|
|
|
|
/** `n` distinct scalar tags. */
|
|
function tags(n: number, prefix = 't'): string[] {
|
|
return Array.from({ length: n }, (_, i) => `${prefix}${i}`)
|
|
}
|
|
|
|
describe('the indexable-array bound', () => {
|
|
let brain: Brainy<any>
|
|
|
|
beforeEach(async () => {
|
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
await brain.init()
|
|
})
|
|
|
|
describe('BELOW the bound: the array indexes, every element of it', () => {
|
|
it('the eleven-element array that used to vanish is searchable', async () => {
|
|
// ELEVEN — one over the old silent limit, the whole shape of the defect.
|
|
await brain.add({
|
|
id: 'eleven',
|
|
data: 'a row with eleven tags',
|
|
type: NounType.Document,
|
|
metadata: { tags: tags(11) },
|
|
vector: []
|
|
})
|
|
|
|
// Every element is a posting, including the eleventh.
|
|
for (const tag of tags(11)) {
|
|
const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
|
|
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('eleven'))
|
|
}
|
|
})
|
|
|
|
it('indexes right up to the bound — every element of it', async () => {
|
|
await brain.add({
|
|
id: 'at-bound',
|
|
data: 'a row at the bound',
|
|
type: NounType.Document,
|
|
metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) },
|
|
vector: []
|
|
})
|
|
|
|
// The first, the last, and one in the middle — all derived from the
|
|
// bound, so the case follows the constant wherever it moves.
|
|
for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, `t${Math.floor(MAX_INDEXED_ARRAY_LENGTH / 2)}`]) {
|
|
const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
|
|
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound'))
|
|
}
|
|
})
|
|
|
|
it('a nested bag\'s array indexes under its dotted address', async () => {
|
|
await brain.add({
|
|
id: 'nested',
|
|
data: 'a row with a nested tag list',
|
|
type: NounType.Document,
|
|
metadata: { facets: { labels: tags(20, 'l') } },
|
|
vector: []
|
|
})
|
|
const hits = await brain.find({ where: { 'facets.labels': 'l19' }, limit: 10 } as any)
|
|
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('nested'))
|
|
})
|
|
})
|
|
|
|
describe('ABOVE the bound: the write is refused, by name', () => {
|
|
const OVER = MAX_INDEXED_ARRAY_LENGTH + 1
|
|
|
|
it('add() throws a typed error naming the field, the length and the bound', async () => {
|
|
const err = await brain
|
|
.add({
|
|
id: 'too-many',
|
|
data: 'a row with too many tags',
|
|
type: NounType.Document,
|
|
metadata: { tags: tags(OVER) },
|
|
vector: []
|
|
} as any)
|
|
.catch((e: any) => e)
|
|
|
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
|
expect(err.field).toBe('tags')
|
|
expect(err.length).toBe(OVER)
|
|
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
|
expect(err.type).toBe('VALIDATION')
|
|
// The message carries all three, and names the cures.
|
|
expect(err.message).toContain('tags')
|
|
expect(err.message).toContain(String(OVER))
|
|
expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
|
expect(err.message).toContain('vector')
|
|
})
|
|
|
|
it('the refused row is not written at all — no half-indexed ghost', async () => {
|
|
await expect(
|
|
brain.add({
|
|
id: 'refused',
|
|
data: 'refused',
|
|
type: NounType.Document,
|
|
metadata: { tags: tags(OVER) },
|
|
vector: []
|
|
} as any)
|
|
).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
|
|
|
|
expect(await brain.get('refused')).toBeNull()
|
|
const hits = await brain.find({ where: { tags: 't0' }, limit: 10 } as any)
|
|
expect(hits.map((r: any) => r.id)).not.toContain(resolveEntityId('refused'))
|
|
})
|
|
|
|
it('a 384-float embedding parked in the metadata bag is refused, not swallowed', async () => {
|
|
const err = await brain
|
|
.add({
|
|
id: 'bag-vector',
|
|
data: 'an embedding in the wrong place',
|
|
type: NounType.Document,
|
|
metadata: { embedding: Array.from({ length: 384 }, (_, i) => i / 384) },
|
|
vector: []
|
|
} as any)
|
|
.catch((e: any) => e)
|
|
|
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
|
expect(err.field).toBe('embedding')
|
|
expect(err.length).toBe(384)
|
|
})
|
|
|
|
it('update() refuses it too', async () => {
|
|
await brain.add({
|
|
id: 'grow',
|
|
data: 'starts small',
|
|
type: NounType.Document,
|
|
metadata: { tags: tags(3) },
|
|
vector: []
|
|
})
|
|
await expect(
|
|
brain.update({ id: 'grow', metadata: { tags: tags(OVER) } } as any)
|
|
).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
|
|
|
|
// And the row keeps the values it had.
|
|
const hits = await brain.find({ where: { tags: 't1' }, limit: 10 } as any)
|
|
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('grow'))
|
|
})
|
|
|
|
it('relate() refuses it on a verb\'s metadata', async () => {
|
|
await brain.add({ id: 'a', data: 'a', type: NounType.Thing, vector: [] })
|
|
await brain.add({ id: 'b', data: 'b', type: NounType.Thing, vector: [] })
|
|
await expect(
|
|
brain.relate({
|
|
from: 'a',
|
|
to: 'b',
|
|
type: VerbType.RelatedTo,
|
|
metadata: { tags: tags(OVER) }
|
|
} as any)
|
|
).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
|
|
})
|
|
|
|
it('a nested oversize array is refused under its dotted address', async () => {
|
|
const err = await brain
|
|
.add({
|
|
id: 'nested-over',
|
|
data: 'nested and too long',
|
|
type: NounType.Document,
|
|
metadata: { facets: { labels: tags(OVER, 'l') } },
|
|
vector: []
|
|
} as any)
|
|
.catch((e: any) => e)
|
|
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
|
expect(err.field).toBe('facets.labels')
|
|
})
|
|
})
|
|
|
|
describe('a row already on disk is admitted, and the skip is NARRATED', () => {
|
|
afterEach(() => {
|
|
vi.restoreAllMocks()
|
|
})
|
|
|
|
it('extraction over an old oversize row warns by field, length and bound', async () => {
|
|
const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
|
|
const index = (brain as any).metadataIndex
|
|
|
|
// The shape an older engine persisted: the write door never saw it, so
|
|
// this reaches extraction directly — exactly as a rebuild or a remove
|
|
// reading the row back would.
|
|
const fields = index.extractIndexableFields({
|
|
metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH + 5), keep: 'me' }
|
|
})
|
|
|
|
// The oversize field contributes nothing...
|
|
expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(0)
|
|
// ...the rest of the row indexes normally — the row is not rejected...
|
|
expect(fields.some((f: any) => f.field === 'keep' && f.value === 'me')).toBe(true)
|
|
// ...and the skip is said out loud, with everything needed to act on it.
|
|
expect(warn).toHaveBeenCalled()
|
|
const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n')
|
|
expect(said).toContain('tags')
|
|
expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH + 5))
|
|
expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
|
expect(said).toContain('NOT indexed')
|
|
})
|
|
|
|
it('an at-bound row on disk is indexed in full and says nothing', async () => {
|
|
const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
|
|
const index = (brain as any).metadataIndex
|
|
|
|
const fields = index.extractIndexableFields({
|
|
metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) }
|
|
})
|
|
expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(MAX_INDEXED_ARRAY_LENGTH)
|
|
|
|
const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n')
|
|
expect(said).not.toContain('indexing bound')
|
|
})
|
|
})
|
|
})
|